diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de11468c2..5d21e35ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -582,11 +582,6 @@ jobs: registry: - test/registry-mocked - test/registry - sdk: - - any - include: - - registry: test/registry-mocked - sdk: omni timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }} steps: @@ -902,7 +897,6 @@ jobs: - name: Run robot mocked functional tests env: PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' - if: success() && matrix.sdk != 'omni' run: | if [ "${{ matrix.registry }}" = "test/registry" ]; then robot \ @@ -916,61 +910,36 @@ jobs: fi - name: Output from mocked functional tests - if: always() && matrix.sdk != 'omni' + if: always() run: | cat ./test/robot/reports/output.xml - # Omni cutover readiness: report the full suite result, then fail CI if it fails. - - name: Omni cutover readiness traffic light - if: matrix.sdk == 'omni' - env: - PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' - # "omni":"all" routes unprefixed providers through omnisdk; "unstable" alone only exposes stackql_unstable_*. - STACKQL_ROBOT_DEFAULT_PREVIEW: '{"unstable":true,"staging":true,"omni":"all"}' + - name: Robot results traffic light + if: always() run: | - pgrep -f flask | xargs kill -9 || true - if python cicd/python/build.py --robot-test --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true" } }'; then - robot_status=0 - else - robot_status=$? - fi if [ -f test/robot/reports/output.xml ]; then - if python test/python/stackql_test_tooling/robot_summary.py \ + python test/python/stackql_test_tooling/robot_summary.py \ --output test/robot/reports/output.xml \ - --title "Omni cutover readiness" \ - --require-100-percent \ - --context "Mocked functional suite, every run opted in with STACKQL_ROBOT_DEFAULT_PREVIEW=${STACKQL_ROBOT_DEFAULT_PREVIEW}" \ - >> "${GITHUB_STEP_SUMMARY}"; then - summary_status=0 - else - summary_status=$? - fi + --title "Robot results" \ + --require-100-percent >> "${GITHUB_STEP_SUMMARY}" else { - echo "# 🔴 Omni cutover readiness" + echo "# 🔴 Robot results" echo "" echo "No robot output was produced; the suite did not run." } >> "${GITHUB_STEP_SUMMARY}" - summary_status=1 - fi - if [ "$robot_status" -ne 0 ] || [ "$summary_status" -ne 0 ]; then exit 1 fi - - name: Output from omni functional tests - if: always() && matrix.sdk == 'omni' - run: | - cat ./test/robot/reports/output.xml - - name: Upload Robot Reports Artifact uses: actions/upload-artifact@v7.0.1 if: always() && github.repository == 'stackql/stackql-devel' && matrix.registry == 'test/registry-mocked' with: - name: stackql_linux_${{ matrix.sdk }}_robot_reports + name: stackql_linux_robot_reports path: test/robot/reports - name: Run robot mocked functional tests with aggressive concurrency - if: success() && matrix.registry != 'test/registry' && matrix.sdk != 'omni' + if: success() && matrix.registry != 'test/registry' run: | echo "## Stray flask apps to be killed before robot tests ##" pgrep -f flask | xargs kill -9 || true @@ -978,12 +947,12 @@ jobs: python cicd/python/build.py --robot-test --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true", "CONCURRENCY_LIMIT": -1 } }' - name: Output from mocked functional tests with aggressive concurrency - if: always() && matrix.registry != 'test/registry' && matrix.sdk != 'omni' + if: always() && matrix.registry != 'test/registry' run: | cat ./test/robot/reports/output.xml - name: Run robot integration tests - if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.registry != 'test/registry' && matrix.sdk != 'omni' + if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.registry != 'test/registry' env: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -996,7 +965,7 @@ jobs: python cicd/python/build.py --robot-test-integration - name: install and test deb package - if: matrix.registry != 'test/registry' && matrix.sdk != 'omni' + if: matrix.registry != 'test/registry' env: PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' IS_SKIP_MCP_TEST: 'true' @@ -1015,7 +984,7 @@ jobs: echo "stackql version: ${stackqlVersion}" - name: Output from mocked deb package functional tests - if: always() && matrix.registry != 'test/registry' && matrix.sdk != 'omni' + if: always() && matrix.registry != 'test/registry' env: PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python' run: | diff --git a/AGENTS.md b/AGENTS.md index ce8496d95..625961c7b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ These guidelines help contributors, human and otherwise, understand and work effectively on this repository. -We also encourage reading [`docs/developer_guide.md`](/docs/developer_guide.md) for further useful information. For deeper understanding of the "brains" of `stackql`, it is worthwhile to consult [the `AGENTS.md` file of `any-sdk`](https://github.com/stackql/any-sdk/blob/main/AGENTS.md) and linked documents from there. +We also encourage reading [`docs/developer_guide.md`](/docs/developer_guide.md) for further useful information. For deeper understanding of the "brains" of `stackql`, it is worthwhile to consult [the omnisdk developer invariants](https://github.com/stackql-labs/omnisdk/blob/main/docs/developer/invariants.md) and linked documents from there. We have upgraded to golang `>= 1.27.3` in order to take advantage of [time simulation and other capabilities in `testing/synctest`](https://go.dev/blog/testing-time). @@ -10,7 +10,7 @@ We have upgraded to golang `>= 1.27.3` in order to take advantage of [time simul ## Project Structure & Module Organization - Entrypoint: [`stackql/main.go`](/stackql/main.go). -- Ideally, foreign system semantics are dealt with in the `any-sdk` repository. +- Ideally, foreign system semantics are dealt with in the `omnisdk` repository. - Loose adherence to popular idioms: - App internals in [`internal`](/internal). - Re-usable modules in [`pkg`](/pkg). @@ -31,7 +31,7 @@ Common commands (run from repo root): ## Coding Style & Naming Conventions -- Program to abstractions; concrete types and foreign-system semantics belong behind interfaces, ideally in `any-sdk`. +- Program to abstractions; concrete types and foreign-system semantics belong behind interfaces, in `omnisdk`. - `gofmt`/`goimports` formatting; `golangci-lint` must pass (see [`.golangci.yml`](/.golangci.yml)). - Go identifier conventions: exported `CamelCase`, unexported `camelCase`, acronyms uppercase (`HTTPClient`, not `HttpClient`). Package names short and lowercase. @@ -43,7 +43,7 @@ Common commands (run from repo root): ## Tools & Resources - Inspect provider surface via the running server: `SHOW PROVIDERS;`, `SHOW SERVICES IN ;`, `DESCRIBE ;`. -- Provider definitions live in the [`stackql-provider-registry`](https://github.com/stackql/stackql-provider-registry); the request-execution brain lives in [`any-sdk`](https://github.com/stackql/any-sdk). +- Provider definitions live in the [`stackql-provider-registry`](https://github.com/stackql/stackql-provider-registry); the request-execution brain lives in [`omnisdk`](https://github.com/stackql-labs/omnisdk). ## Commit & Pull Request Guidelines @@ -61,15 +61,16 @@ StackQL supports flexible configuration management, including authentication sec ## SQLite backend - The embedded backend is `modernc.org/sqlite` (pure Go, no cgo), pinned >= v1.57.0 - (caller-constructed `Driver` function registration). It is consumed via - [`any-sdk`](https://github.com/stackql/any-sdk) `public/sqlengine`, which registers + (caller-constructed `Driver` function registration). It is owned by + `internal/stackql/sqlengine`, which registers the driver under the name `stackql-sqlite`. -- DSN construction happens only via the central `BuildDSN` in any-sdk `public/sqlengine` +- SQL connections, staging, and backend execution remain in StackQL; Omni contains no SQL staging. +- DSN construction happens only via the central `BuildDSN` in `internal/stackql/sqlengine` (legacy mattn-style params such as `_busy_timeout=5000` are translated to modernc `_pragma=busy_timeout(5000)` syntax there); driver errors are inspected only via the error predicates in that package (`isBusy`, `isConstraintViolation`); the custom extension functions (`split_part`, `regexp_like`, `regexp_substr`, `regexp_replace`, - `json_equal`, `aws_policy_equal`) live only in any-sdk `public/sqlfuncs` and are + `json_equal`, `aws_policy_equal`) live only in `internal/stackql/sqlengine/internal/sqlfuncs` and are registered on the constructed driver, never globally. - Never reintroduce `mattn/go-sqlite3`, cgo, the `sqlite_stackql` build tag, or package-global function registration. All builds and release artifacts are diff --git a/cicd/go/dependency-metadata/main.go b/cicd/go/dependency-metadata/main.go index f173222f0..9e02a8cc7 100644 --- a/cicd/go/dependency-metadata/main.go +++ b/cicd/go/dependency-metadata/main.go @@ -198,7 +198,7 @@ func description(path string) string { switch path { case "github.com/stackql/stackql-parser": return "StackQL SQL parser, derived from Vitess" - case "github.com/stackql/any-sdk": + case "github.com/stackql-labs/omnisdk": return "StackQL provider SDK" case "github.com/stackql/readline": return "StackQL fork of chzyer/readline for interactive line editing" diff --git a/go.mod b/go.mod index d8105ea48..a9a73473a 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.27.3 require ( github.com/DATA-DOG/go-sqlmock v1.5.1 + github.com/Masterminds/semver v1.4.2 + github.com/PaesslerAG/gval v1.0.0 + github.com/PaesslerAG/jsonpath v0.1.1 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e - github.com/getkin/kin-openapi v0.88.0 github.com/google/go-jsonnet v0.17.0 github.com/google/licensecheck v0.3.1 github.com/google/uuid v1.6.0 @@ -20,37 +22,33 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.7-alpha02 - github.com/stackql/any-sdk v0.6.1-beta02 + github.com/stackql-labs/omnisdk v0.1.7-beta05 github.com/stackql/go-suffix-map v0.0.1-alpha02 github.com/stackql/psql-wire v0.1.2-beta02 github.com/stackql/stackql-parser v0.0.16-beta01 + github.com/stackql/stackql-provider-registry v0.0.2-alpha01 github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01 github.com/stretchr/testify v1.11.1 + github.com/xo/dburl v0.23.2 golang.org/x/mod v0.38.0 golang.org/x/sync v0.22.0 gonum.org/v1/gonum v0.17.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible + modernc.org/sqlite v1.59.0 ) require ( - cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/BurntSushi/toml v1.4.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect - github.com/Masterminds/semver v1.4.2 // indirect - github.com/PaesslerAG/gval v1.0.0 // indirect - github.com/PaesslerAG/jsonpath v0.1.1 // indirect - github.com/antchfx/xmlquery v1.3.10 // indirect - github.com/antchfx/xpath v1.3.6 // indirect github.com/apache/arrow-go/v18 v18.0.0 // indirect github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect @@ -83,21 +81,15 @@ require ( github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fullstorydev/grpcurl v1.9.3 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/swag v0.21.1 // indirect github.com/goccy/go-json v0.10.4 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/gofrs/flock v0.10.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/glog v1.2.5 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v24.12.23+incompatible // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/jsonschema-go v0.4.3 // indirect - github.com/gorilla/mux v1.8.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -107,17 +99,13 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jhump/protoreflect v1.18.0 // indirect github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.7 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/oracle/oci-go-sdk/v65 v65.120.0 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect @@ -127,20 +115,16 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect - github.com/sony/gobreaker/v2 v2.4.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect - github.com/stackql/stackql-provider-registry v0.0.2-alpha01 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/xo/dburl v0.23.2 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect @@ -162,11 +146,9 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect - modernc.org/sqlite v1.59.0 // indirect ) replace github.com/chzyer/readline => github.com/stackql/readline v0.0.2-alpha05 diff --git a/go.sum b/go.sum index fa0df8a6f..027908b46 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= @@ -8,14 +6,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WW github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 h1:1mvYtZfWQAnwNah/C+Z+Jb9rQH95LPE2vlmMuWAHJk8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1/go.mod h1:75I/mXtme1JyWFtz8GocPHVFyH421IBoZErnO16dd0k= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 h1:Bk5uOhSAenHyR5P61D/NzeQCv+4fEVV8mOkJ82NqpWw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1/go.mod h1:QZ4pw3or1WPmRBxf0cHd1tknzrT54WPBOQoGutCPvSU= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= -github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= -github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -35,11 +29,6 @@ github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEs github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/antchfx/xmlquery v1.3.10 h1:U2yMwr8U0KmGM2iDG2Ky/3LfxNsiK4uw1bSBkeMO9+g= -github.com/antchfx/xmlquery v1.3.10/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= -github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= -github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/apache/arrow-go/v18 v18.0.0 h1:1dBDaSbH3LtulTyOVYaBCHO3yVRwjV+TZaqn3g6V7ZM= github.com/apache/arrow-go/v18 v18.0.0/go.mod h1:t6+cWRSmKgdQ6HsxisQjok+jBpKGhRDiqcf3p0p/F+A= github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= @@ -101,14 +90,11 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= @@ -124,10 +110,6 @@ github.com/fullstorydev/grpcurl v1.9.3 h1:PC1Xi3w+JAvEE2Tg2Gf2RfVgPbf9+tbuQr1Zky github.com/fullstorydev/grpcurl v1.9.3/go.mod h1:/b4Wxe8bG6ndAjlfSUjwseQReUDUvBJiFEB7UllOlUE= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= -github.com/getkin/kin-openapi v0.88.0 h1:BjJ2JERWJbYE1o1RGEj/5LmR5qw7ecfl3O3su4ImR+0= -github.com/getkin/kin-openapi v0.88.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -136,26 +118,16 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= -github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -175,8 +147,6 @@ github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -239,10 +209,6 @@ github.com/jhump/protoreflect v1.18.0 h1:TOz0MSR/0JOZ5kECB/0ufGnC2jdsgZ123Rd/k4Z github.com/jhump/protoreflect v1.18.0/go.mod h1:ezWcltJIVF4zYdIFM+D/sHV4Oh5LNU08ORzCGfwvTz8= github.com/jhump/protoreflect/v2 v2.0.0-beta.1 h1:Dw1rslK/VotaUGYsv53XVWITr+5RCPXfvvlGrM/+B6w= github.com/jhump/protoreflect/v2 v2.0.0-beta.1/go.mod h1:D9LBEowZyv8/iSu97FU2zmXG3JxVTmNw21mu63niFzU= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= -github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= @@ -271,11 +237,6 @@ github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -303,8 +264,6 @@ github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJm github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4 h1:Mm4XQCBICntJzH8fKglsRuEiFUJYnTnM4BBFvpP5BWs= github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/oracle/oci-go-sdk/v65 v65.120.0 h1:qpGdts2Yleg6TdmtxXkL8MAnsASO+SOG+iG/Nd8wUGk= -github.com/oracle/oci-go-sdk/v65 v65.120.0/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 h1:KPpdlQLZcHfTMQRi6bFQ7ogNO0ltFT4PmtwTLW4W+14= @@ -320,8 +279,6 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= -github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= @@ -333,8 +290,6 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= @@ -350,8 +305,6 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/snowflakedb/gosnowflake v1.15.0 h1:1V4dG1EmJ9O81Hv8y1LAE9koZebmx4tnRAPKWvDf8xA= github.com/snowflakedb/gosnowflake v1.15.0/go.mod h1:+3Eh8swS12G6Fbt/wb5Vcse2Id7VU9HGgKSH8ydiumU= -github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= -github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= @@ -367,10 +320,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.7-alpha02 h1:+V2JYbLe9C6eeiGXXhSC3bIOnKriyt61kAxd2hM8uls= -github.com/stackql-labs/omnisdk v0.1.7-alpha02/go.mod h1:P3pr9kvDTMox+4xCLcFdzOS1wMMe3ho9i6j2Xl+/Yog= -github.com/stackql/any-sdk v0.6.1-beta02 h1:O+PiVsbjMh2165p6ek5vD7xnO+8ji7zxPpT2rEtAHos= -github.com/stackql/any-sdk v0.6.1-beta02/go.mod h1:M9b57jwOLa7afqZE5ZSTzN3YnaRGslJBIAdD5sz83Nw= +github.com/stackql-labs/omnisdk v0.1.7-beta05 h1:Ld2EM1MxtLhXaadO5xoWNPXt97VR1TXdeFELzuR6vvk= +github.com/stackql-labs/omnisdk v0.1.7-beta05/go.mod h1:P3pr9kvDTMox+4xCLcFdzOS1wMMe3ho9i6j2Xl+/Yog= github.com/stackql/go-suffix-map v0.0.1-alpha02 h1:JI4aiq/OfHW4a/BiNqK31NbyaR94zlUOhGZof3G/OUo= github.com/stackql/go-suffix-map v0.0.1-alpha02/go.mod h1:6BSMz4K/gIYfqdVRb7V8xL165wiPYyCjD9Kx7hLMd9Y= github.com/stackql/psql-wire v0.1.2-beta02 h1:7DfliBpyyCaUHVJ1Jxk37q3zVwVLyh6+FCHeM0/gbAQ= @@ -392,7 +343,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= @@ -402,8 +352,6 @@ github.com/xo/dburl v0.23.2 h1:Fl88cvayrgE56JA/sqhNMLljCW/b7RmG1mMkKMZUFgA= github.com/xo/dburl v0.23.2/go.mod h1:uazlaAQxj4gkshhfuuYyvwCBouOmNnG2aDxTCFZpmL4= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -467,7 +415,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= @@ -488,7 +435,6 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -544,7 +490,6 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -557,11 +502,9 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/stackql/acid/tsm_physio/coordinator.go b/internal/stackql/acid/tsm_physio/coordinator.go index d2d7b7766..d246141ef 100644 --- a/internal/stackql/acid/tsm_physio/coordinator.go +++ b/internal/stackql/acid/tsm_physio/coordinator.go @@ -1,9 +1,9 @@ package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature import ( - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/stackql/internal/stackql/acid/acid_dto" "github.com/stackql/stackql/internal/stackql/acid/tsm" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/handler" ) diff --git a/internal/stackql/acid/tsm_physio/txn_orchestrator.go b/internal/stackql/acid/tsm_physio/txn_orchestrator.go index bd48f96aa..8450aaa57 100644 --- a/internal/stackql/acid/tsm_physio/txn_orchestrator.go +++ b/internal/stackql/acid/tsm_physio/txn_orchestrator.go @@ -3,10 +3,10 @@ package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature import ( "fmt" - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/tsm" "github.com/stackql/stackql/internal/stackql/acid/txn_context" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" ) diff --git a/internal/stackql/astanalysis/annotatedast/annotated_ast.go b/internal/stackql/astanalysis/annotatedast/annotated_ast.go index e49f83085..c6f924113 100644 --- a/internal/stackql/astanalysis/annotatedast/annotated_ast.go +++ b/internal/stackql/astanalysis/annotatedast/annotated_ast.go @@ -2,7 +2,6 @@ package annotatedast import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/astanalysis/selectmetadata" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/parserutil" @@ -15,7 +14,6 @@ func NewAnnotatedAst(parent AnnotatedAst, ast sqlparser.Statement) (AnnotatedAst tableIndirects: make(map[string]astindirect.Indirect), materializedViewRefs: make(map[string]astindirect.Indirect), tableSQLDataSources: make(map[string]sql_datasource.SQLDataSource), - selectMetadataMap: make(map[*sqlparser.Select]selectmetadata.SelectMetadata), whereParamMaps: make(map[*sqlparser.Where]parserutil.ParameterMap), insertRowsIndirect: make(map[*sqlparser.Insert]astindirect.Indirect), selectIndirectCache: make(map[*sqlparser.Select]astindirect.Indirect), @@ -32,12 +30,10 @@ type AnnotatedAst interface { GetIndirects() map[string]astindirect.Indirect GetMaterializedView(sqlparser.SQLNode) (astindirect.Indirect, bool) GetPhysicalTable(sqlparser.SQLNode) (astindirect.Indirect, bool) - GetSelectMetadata(*sqlparser.Select) (selectmetadata.SelectMetadata, bool) GetSQLDataSource(node sqlparser.SQLNode) (sql_datasource.SQLDataSource, bool) SetIndirect(node sqlparser.SQLNode, indirect astindirect.Indirect) SetMaterializedView(node sqlparser.SQLNode, indirect astindirect.Indirect) SetPhysicalTable(node sqlparser.SQLNode, indirect astindirect.Indirect) - SetSelectMetadata(*sqlparser.Select, selectmetadata.SelectMetadata) SetSQLDataSource(node sqlparser.SQLNode, sqlDataSource sql_datasource.SQLDataSource) SetWhereParamMapsEntry(*sqlparser.Where, parserutil.ParameterMap) GetWhereParamMapsEntry(*sqlparser.Where) (parserutil.ParameterMap, bool) @@ -59,7 +55,6 @@ type standardAnnotatedAst struct { physicalTableRefs map[string]astindirect.Indirect insertRowsIndirect map[*sqlparser.Insert]astindirect.Indirect tableSQLDataSources map[string]sql_datasource.SQLDataSource - selectMetadataMap map[*sqlparser.Select]selectmetadata.SelectMetadata whereParamMaps map[*sqlparser.Where]parserutil.ParameterMap selectIndirectCache map[*sqlparser.Select]astindirect.Indirect unionIndirectCache map[*sqlparser.Union]astindirect.Indirect @@ -160,15 +155,6 @@ func (aa *standardAnnotatedAst) IsReadOnly() bool { return false } -func (aa *standardAnnotatedAst) GetSelectMetadata(selNode *sqlparser.Select) (selectmetadata.SelectMetadata, bool) { - sm, ok := aa.selectMetadataMap[selNode] - return sm, ok -} - -func (aa *standardAnnotatedAst) SetSelectMetadata(selNode *sqlparser.Select, meta selectmetadata.SelectMetadata) { - aa.selectMetadataMap[selNode] = meta -} - func (aa *standardAnnotatedAst) GetIndirect(node sqlparser.SQLNode) (astindirect.Indirect, bool) { switch n := node.(type) { case *sqlparser.AliasedTableExpr: diff --git a/internal/stackql/astanalysis/annotatedast/annotatedast_test.go b/internal/stackql/astanalysis/annotatedast/annotatedast_test.go index 056566f3c..1f95ffb7b 100644 --- a/internal/stackql/astanalysis/annotatedast/annotatedast_test.go +++ b/internal/stackql/astanalysis/annotatedast/annotatedast_test.go @@ -96,11 +96,6 @@ func TestAnnotatedAst_GettersAndSetters(t *testing.T) { _, ok = aa.GetWhereParamMapsEntry(where) assert.True(t, ok) - // Select metadata - aa.SetSelectMetadata(sel, nil) - _, ok = aa.GetSelectMetadata(sel) - assert.True(t, ok) - // Read only assert.False(t, aa.IsReadOnly()) } diff --git a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go index 6e25dc985..3a1d03b7a 100644 --- a/internal/stackql/astanalysis/earlyanalysis/ast_expand.go +++ b/internal/stackql/astanalysis/earlyanalysis/ast_expand.go @@ -4,11 +4,11 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/primitivebuilder" "github.com/stackql/stackql/internal/stackql/primitivegenerator" @@ -212,10 +212,6 @@ func (v *indirectExpandAstVisitor) processIndirect(node sqlparser.SQLNode, indir } indirect.SetUnderlyingSymTab(underlyingSymTab) indirect.SetSelectContext(indirectPrimitiveGenerator.GetPrimitiveComposer().GetIndirectSelectPreparedStatementCtx()) - assignedParams, ok := indirectPrimitiveGenerator.GetPrimitiveComposer().GetAssignedParameters() - if ok { - indirect.SetAssignedParameters(assignedParams) - } createBuilder, createBuilderExists := indirectPrimitiveGenerator.GetIndirectCreateTailBuilder() if createBuilderExists { v.createBuilder = append(v.createBuilder, createBuilder...) @@ -915,8 +911,6 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error { } } providerName := node.QualifierSecond.GetRawVal() - serviceName := node.Qualifier.GetRawVal() - resourceName := node.Name.GetRawVal() sqlDataSource, hasSQLDataSource := v.handlerCtx.GetSQLDataSource(providerName) if hasSQLDataSource { // Persist data source for later retrieval @@ -928,17 +922,10 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error { //nolint:nestif // deferring cosmetics on visitors if err != nil { logging.GetLogger().Debugf("optimistic doc error: %s", err.Error()) - } else { - if hasSQLDataSource { - err = prov.PersistStaticExternalSQLDataSource(v.handlerCtx.GetRuntimeContext()) - if err != nil { - return err - } - } else { - _, err = prov.GetServiceShard(serviceName, resourceName, v.handlerCtx.GetRuntimeContext()) - if err != nil { - logging.GetLogger().Debugf("optimistic doc error: %s", err.Error()) - } + } else if hasSQLDataSource { + err = prov.PersistStaticExternalSQLDataSource(v.handlerCtx.GetRuntimeContext()) + if err != nil { + return err } } // END OPTIMISTIC DOC PERSISTENCE AND VIEW DEFINITION diff --git a/internal/stackql/astanalysis/earlyanalysis/constant_fold.go b/internal/stackql/astanalysis/earlyanalysis/constant_fold.go index 7d43f836a..67730dc62 100644 --- a/internal/stackql/astanalysis/earlyanalysis/constant_fold.go +++ b/internal/stackql/astanalysis/earlyanalysis/constant_fold.go @@ -3,8 +3,8 @@ package earlyanalysis import ( "fmt" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/sqlengine" ) // Constant folding for WHERE-clause function expressions (issue #686). diff --git a/internal/stackql/astanalysis/earlyanalysis/first_passes.go b/internal/stackql/astanalysis/earlyanalysis/first_passes.go index ee0e54d1e..92c2c3316 100644 --- a/internal/stackql/astanalysis/earlyanalysis/first_passes.go +++ b/internal/stackql/astanalysis/earlyanalysis/first_passes.go @@ -3,15 +3,13 @@ package earlyanalysis //nolint:cyclop // analysers are inherently complex, for n import ( "fmt" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/astanalysis/routeanalysis" "github.com/stackql/stackql/internal/stackql/astvisit" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/intrinsic" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/planbuilderinput" "github.com/stackql/stackql/internal/stackql/primitivebuilder" @@ -28,10 +26,6 @@ const ( NopInstruction ) -var ( - errPgOnly error = fmt.Errorf("cannot accomodate PG-only statement when backend is not matched to PG") -) - type Analyzer interface { Analyze(statement sqlparser.Statement, handlerCtx handler.HandlerContext, tcc internaldto.TxnControlCounters) error } @@ -414,34 +408,7 @@ func (sp *standardInitialPasses) initialPasses( return fmt.Errorf("expected DDL statement in analysing 'create materialized view' statement") } } - switch node := astToAnalyse.(type) { - case *sqlparser.Select: - routeAnalyzer := routeanalysis.NewSelectRoutePass(node, pbi, whereParams) - err = routeAnalyzer.RoutePass() - if err != nil { - return err - } - if routeAnalyzer.IsPGInternalOnly() { - if sp.primitiveGenerator.GetPrimitiveComposer().GetSQLSystem().GetName() != constants.SQLDialectPostgres { - return errPgOnly - } - sp.instructionType = InternallyRoutableInstruction - } - pbi = routeAnalyzer.GetPlanBuilderInput() - case *sqlparser.ParenSelect: - routeAnalyzer := routeanalysis.NewSelectRoutePass(node.Select, pbi, whereParams) - err = routeAnalyzer.RoutePass() - if err != nil { - return err - } - if routeAnalyzer.IsPGInternalOnly() { - if sp.primitiveGenerator.GetPrimitiveComposer().GetSQLSystem().GetName() != constants.SQLDialectPostgres { - return errPgOnly - } - sp.instructionType = InternallyRoutableInstruction - } - pbi = routeAnalyzer.GetPlanBuilderInput() - case *sqlparser.Union: + if node, isUnion := astToAnalyse.(*sqlparser.Union); isUnion { lhsThreeToFiveAgg, passErr := thirdToFifthPasses(node.FirstStatement, annotatedAST) if passErr != nil { return passErr @@ -484,18 +451,7 @@ func (sp *standardInitialPasses) initialPasses( rhsPbi.WithNext(nextPbi) rhsPbi = nextPbi } - routeAnalyzer := routeanalysis.NewSelectRoutePass(node, lhsPbi, whereParams) - err = routeAnalyzer.RoutePass() - if err != nil { - return err - } - if routeAnalyzer.IsPGInternalOnly() { - if sp.primitiveGenerator.GetPrimitiveComposer().GetSQLSystem().GetName() != constants.SQLDialectPostgres { - return errPgOnly - } - sp.instructionType = InternallyRoutableInstruction - } - pbi = routeAnalyzer.GetPlanBuilderInput() + pbi = lhsPbi } pbi.SetCreateMaterializedView(isCreateMAterializedView) diff --git a/internal/stackql/astanalysis/routeanalysis/select_route_analysis.go b/internal/stackql/astanalysis/routeanalysis/select_route_analysis.go deleted file mode 100644 index 55fd22c74..000000000 --- a/internal/stackql/astanalysis/routeanalysis/select_route_analysis.go +++ /dev/null @@ -1,187 +0,0 @@ -package routeanalysis - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/astanalysis/selectmetadata" - "github.com/stackql/stackql/internal/stackql/astvisit" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/planbuilderinput" - "github.com/stackql/stackql/internal/stackql/router" -) - -var ( - _ RoutePass = &standardSelectRoutePass{} -) - -// TODO: must accomodate parent (and indeed ancestor) clause parameter passing. -type RoutePass interface { - RoutePass() error - GetPlanBuilderInput() planbuilderinput.PlanBuilderInput - IsPGInternalOnly() bool -} - -func NewSelectRoutePass( - node sqlparser.SelectStatement, - pbi planbuilderinput.PlanBuilderInput, - parentWhereParams parserutil.ParameterMap, -) RoutePass { - return &standardSelectRoutePass{ - inputPbi: pbi, - handlerCtx: pbi.GetHandlerCtx(), - node: node, - parentWhereParams: parentWhereParams, - } -} - -type standardSelectRoutePass struct { - inputPbi planbuilderinput.PlanBuilderInput - outputPbi planbuilderinput.PlanBuilderInput - handlerCtx handler.HandlerContext - node sqlparser.SelectStatement - parentWhereParams parserutil.ParameterMap - isPGInternalOnly bool -} - -func (sp *standardSelectRoutePass) IsPGInternalOnly() bool { - return sp.isPGInternalOnly -} - -func (sp *standardSelectRoutePass) GetPlanBuilderInput() planbuilderinput.PlanBuilderInput { - return sp.outputPbi -} - -//nolint:funlen,gocognit // defer uplifts on analysers -func (sp *standardSelectRoutePass) RoutePass() error { - var node *sqlparser.Select - - pbi := sp.inputPbi.Clone() - - // counters := pbi.GetTxnCtrlCtrs() - - switch n := sp.node.(type) { - case *sqlparser.Select: - node = n - case *sqlparser.ParenSelect: - routePass := NewSelectRoutePass(n.Select, sp.inputPbi, sp.parentWhereParams) - err := routePass.RoutePass() - sp.isPGInternalOnly = routePass.IsPGInternalOnly() - sp.outputPbi = pbi - return err - case *sqlparser.Union: - routePass := NewSelectRoutePass(n.FirstStatement, pbi, sp.parentWhereParams) - err := routePass.RoutePass() - if err != nil { - return err - } - lhsPGInternalOnly := routePass.IsPGInternalOnly() - // TODO: eventualy accomodate sharing pg native stuff to - // mix and match with stackql stuff. - var rhsNonPGInternalDetected bool - rhsPbi := pbi - var hasPbi bool - for _, s := range n.UnionSelects { - rhsPbi, hasPbi = rhsPbi.Next() - if !hasPbi { - return fmt.Errorf("no more PBIs for union selects") - } - // ctrClone := counters.CloneAndIncrementInsertID() - // rhsPbi.SetTxnCtrlCtrs(ctrClone) - routePass := NewSelectRoutePass(s.Statement, rhsPbi, sp.parentWhereParams) //nolint:govet // intentional shadow - err = routePass.RoutePass() - if err != nil { - return err - } - if !routePass.IsPGInternalOnly() { - rhsNonPGInternalDetected = true - } - } - - sp.isPGInternalOnly = lhsPGInternalOnly && !rhsNonPGInternalDetected && len(n.UnionSelects) > 0 - sp.outputPbi = pbi - return nil - } - - handlerCtx := sp.handlerCtx - - annotatedAST := pbi.GetAnnotatedAST() - - // TODO: get rid of this and dependent tests. - // We need not emulate postgres for other backends at this stage. - if sel, ok := planbuilderinput.IsPGSetupQuery(pbi); ok { - if sel != nil { - return nil - } - return nil - } - - // BLOCK ParameterHierarchy - // The AST analysis passes extract parameters - // prior to the assembly of hierarchies. - // This is a chicken and egg scenario: - // - we need hierarchies a priori for temporal - // dependencies between tables. - // - we need parameters to determine hierarchy (for now). - // - parameters may refer to tables and we want to reference - // this for semantic analysis and later temporal sequencing, - // data flow semantics. - // - TODO: so... will need to split this up into multiple passes; - // parameters will need to have Hierarchies attached after they are inferred. - // Then semantic anlaysis and data flow can be instrumented. - // - TODO: add support for views and subqueries. - whereParamMap, ok := pbi.GetAnnotatedAST().GetWhereParamMapsEntry(node.Where) - if !ok { - whereParamMap = astvisit.ExtractParamsFromWhereClause(annotatedAST, node.Where) - } - whereParamMap.Merge(sp.parentWhereParams) - onParamMap := astvisit.ExtractParamsFromFromClause(annotatedAST, node.From) - - // TODO: There is god awful object <-> namespacing inside here: abstract it. - paramRouter := router.NewParameterRouter( - annotatedAST, - pbi.GetAliasedTables(), - pbi.GetAssignedAliasedColumns(), - whereParamMap, - onParamMap, - pbi.GetColRefs(), - handlerCtx.GetNamespaceCollection(), - handlerCtx.GetASTFormatter(), - handlerCtx.GetDataFlowCfg(), - ) - - // TODO: Do the proper SOLID treatment on router, etc. - // Might need to split into multiple passes. - v := router.NewTableRouteAstVisitor(pbi.GetHandlerCtx(), paramRouter, false) - - err := v.Visit(node) - - if err != nil { - return err - } - - pbi = pbi.WithParameterRouter(paramRouter) - - pbi = pbi.WithTableRouteVisitor(v) - - onConditionsToRewrite := v.GetParameterRouter().GetOnConditionsToRewrite() - // TODO: ensure this only contains actual data flows, not normal on conitions - onConditionDataFlows, err := v.GetParameterRouter().GetOnConditionDataFlows() - if err != nil { - return err - } - - selectMetadata := selectmetadata.NewSelectMetadata( - onConditionDataFlows, - onConditionsToRewrite, - v.GetTableMap(), - v.GetAnnotations(), - ) - - annotatedAST.SetSelectMetadata(node, selectMetadata) - - sp.outputPbi = pbi - - return nil -} diff --git a/internal/stackql/astanalysis/selectmetadata/select_metadata.go b/internal/stackql/astanalysis/selectmetadata/select_metadata.go deleted file mode 100644 index 0528923a3..000000000 --- a/internal/stackql/astanalysis/selectmetadata/select_metadata.go +++ /dev/null @@ -1,55 +0,0 @@ -package selectmetadata - -import ( - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/dataflow" - "github.com/stackql/stackql/internal/stackql/taxonomy" -) - -var ( - _ SelectMetadata = &standardSelectMetadata{} -) - -func NewSelectMetadata( - onConditionDataFlows dataflow.Collection, - onConditionsToRewrite map[*sqlparser.ComparisonExpr]struct{}, - tableMap taxonomy.TblMap, - annotations taxonomy.AnnotationCtxMap, -) SelectMetadata { - return &standardSelectMetadata{ - onConditionDataFlows: onConditionDataFlows, - onConditionsToRewrite: onConditionsToRewrite, - tableMap: tableMap, - annotations: annotations, - } -} - -type SelectMetadata interface { - GetAnnotations() (taxonomy.AnnotationCtxMap, bool) - GetOnConditionDataFlows() (dataflow.Collection, bool) - GetOnConditionsToRewrite() map[*sqlparser.ComparisonExpr]struct{} - GetTableMap() (taxonomy.TblMap, bool) -} - -type standardSelectMetadata struct { - onConditionDataFlows dataflow.Collection - onConditionsToRewrite map[*sqlparser.ComparisonExpr]struct{} - tableMap taxonomy.TblMap - annotations taxonomy.AnnotationCtxMap -} - -func (sm *standardSelectMetadata) GetTableMap() (taxonomy.TblMap, bool) { - return sm.tableMap, sm.tableMap != nil -} - -func (sm *standardSelectMetadata) GetAnnotations() (taxonomy.AnnotationCtxMap, bool) { - return sm.annotations, sm.annotations != nil -} - -func (sm *standardSelectMetadata) GetOnConditionsToRewrite() map[*sqlparser.ComparisonExpr]struct{} { - return sm.onConditionsToRewrite -} - -func (sm *standardSelectMetadata) GetOnConditionDataFlows() (dataflow.Collection, bool) { - return sm.onConditionDataFlows, sm.onConditionDataFlows != nil -} diff --git a/internal/stackql/astanalysis/selectmetadata/select_metadata_test.go b/internal/stackql/astanalysis/selectmetadata/select_metadata_test.go deleted file mode 100644 index 0042fc45d..000000000 --- a/internal/stackql/astanalysis/selectmetadata/select_metadata_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package selectmetadata //nolint:testpackage // to test unexported methods - -import ( - "testing" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/dataflow" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stretchr/testify/assert" -) - -func TestSelectMetadata(t *testing.T) { - var onConditionDataFlows dataflow.Collection - onConditionsToRewrite := make(map[*sqlparser.ComparisonExpr]struct{}) - tableMap := taxonomy.TblMap{} - annotations := taxonomy.AnnotationCtxMap{} - t.Run("NewSelectMetadata", func(t *testing.T) { - sm := NewSelectMetadata(onConditionDataFlows, onConditionsToRewrite, tableMap, annotations) - assert.NotNil(t, sm) - }) - - t.Run("GetTableMap", func(t *testing.T) { - sm := NewSelectMetadata(onConditionDataFlows, onConditionsToRewrite, tableMap, annotations) - val, exp := sm.GetTableMap() - assert.Equal(t, val, tableMap) - assert.Equal(t, exp, true) - }) - - t.Run("GetAnnotations", func(t *testing.T) { - sm := NewSelectMetadata(onConditionDataFlows, onConditionsToRewrite, tableMap, annotations) - val, exp := sm.GetAnnotations() - assert.Equal(t, val, annotations) - assert.Equal(t, exp, true) - }) - - t.Run("GetOnConditionsToRewrite", func(t *testing.T) { - sm := NewSelectMetadata(onConditionDataFlows, onConditionsToRewrite, tableMap, annotations) - val := sm.GetOnConditionsToRewrite() - assert.Equal(t, val, onConditionsToRewrite) - }) - - t.Run("GetOnConditionDataFlows", func(t *testing.T) { - sm := NewSelectMetadata(onConditionDataFlows, onConditionsToRewrite, tableMap, annotations) - val, exp := sm.GetOnConditionDataFlows() - assert.Equal(t, val, onConditionDataFlows) - assert.Equal(t, exp, false) - }) -} diff --git a/internal/stackql/astfuncrewrite/astfuncrewrite.go b/internal/stackql/astfuncrewrite/astfuncrewrite.go index a0c2e09dd..a141bd33b 100644 --- a/internal/stackql/astfuncrewrite/astfuncrewrite.go +++ b/internal/stackql/astfuncrewrite/astfuncrewrite.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/constants" ) const ( diff --git a/internal/stackql/astindirect/cte_indirect.go b/internal/stackql/astindirect/cte_indirect.go index 7b78a5446..1d8197bff 100644 --- a/internal/stackql/astindirect/cte_indirect.go +++ b/internal/stackql/astindirect/cte_indirect.go @@ -1,10 +1,8 @@ package astindirect import ( - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/symtab" "github.com/stackql/stackql/internal/stackql/typing" ) @@ -15,7 +13,6 @@ type CTE struct { name string selectStmt sqlparser.SelectStatement selCtx drm.PreparedStatementCtx - paramCollection internaldto.TableParameterCollection underlyingSymbolTable symtab.SymTab next Indirect isElide bool @@ -84,14 +81,6 @@ func (c *CTE) GetRelationalColumns() []typing.RelationalColumn { return nil } -func (c *CTE) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return c.paramCollection, c.paramCollection != nil -} - -func (c *CTE) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - c.paramCollection = paramCollection -} - func (c *CTE) GetUnderlyingSymTab() symtab.SymTab { return c.underlyingSymbolTable } @@ -117,14 +106,6 @@ func (c *CTE) GetColumns() []typing.ColumnMetadata { return cols } -func (c *CTE) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (c *CTE) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (c *CTE) GetColumnByName(name string) (typing.ColumnMetadata, bool) { if c.selCtx != nil { for _, col := range c.selCtx.GetNonControlColumns() { diff --git a/internal/stackql/astindirect/indirect.go b/internal/stackql/astindirect/indirect.go index f64c56e18..7329c8cf9 100644 --- a/internal/stackql/astindirect/indirect.go +++ b/internal/stackql/astindirect/indirect.go @@ -3,7 +3,6 @@ package astindirect import ( "fmt" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/sql_system" "github.com/stackql/stackql/internal/stackql/symtab" @@ -101,19 +100,18 @@ func NewSubqueryIndirect(subQueryDTO internaldto.SubqueryDTO) (Indirect, error) type Indirect interface { Parse() error - GetAssignedParameters() (internaldto.TableParameterCollection, bool) + GetColumnByName(name string) (typing.ColumnMetadata, bool) GetRelationalColumnByIdentifier(name string) (typing.RelationalColumn, bool) GetColumns() []typing.ColumnMetadata GetRelationalColumns() []typing.RelationalColumn GetName() string - GetOptionalParameters() map[string]formulation.Addressable - GetRequiredParameters() map[string]formulation.Addressable + GetSelectAST() sqlparser.SelectStatement GetSelectContext() drm.PreparedStatementCtx GetType() IndirectType GetUnderlyingSymTab() symtab.SymTab - SetAssignedParameters(internaldto.TableParameterCollection) + SetSelectContext(drm.PreparedStatementCtx) SetUnderlyingSymTab(symtab.SymTab) GetTranslatedDDL() (string, bool) diff --git a/internal/stackql/astindirect/materialized_view_indirect.go b/internal/stackql/astindirect/materialized_view_indirect.go index c93959a77..a5a0b7e4f 100644 --- a/internal/stackql/astindirect/materialized_view_indirect.go +++ b/internal/stackql/astindirect/materialized_view_indirect.go @@ -4,7 +4,6 @@ package astindirect import ( "fmt" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -17,7 +16,6 @@ import ( type MaterializedView struct { viewDTO internaldto.RelationDTO selectStmt sqlparser.SelectStatement - paramCollection internaldto.TableParameterCollection sqlSystem sql_system.SQLSystem underlyingSymbolTable symtab.SymTab } @@ -46,14 +44,6 @@ func (v *MaterializedView) GetType() IndirectType { return MaterializedViewType } -func (v *MaterializedView) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return v.paramCollection, v.paramCollection != nil -} - -func (v *MaterializedView) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - v.paramCollection = paramCollection -} - func (v *MaterializedView) GetUnderlyingSymTab() symtab.SymTab { return v.underlyingSymbolTable } @@ -86,14 +76,6 @@ func (v *MaterializedView) GetRelationalColumnByIdentifier(name string) (typing. return nil, false } -func (v *MaterializedView) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (v *MaterializedView) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (v *MaterializedView) GetColumnByName(_ string) (typing.ColumnMetadata, bool) { return nil, false } diff --git a/internal/stackql/astindirect/physical_table_indirect.go b/internal/stackql/astindirect/physical_table_indirect.go index 3e7a02b6a..a5b9220d3 100644 --- a/internal/stackql/astindirect/physical_table_indirect.go +++ b/internal/stackql/astindirect/physical_table_indirect.go @@ -4,7 +4,6 @@ package astindirect import ( "fmt" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -17,7 +16,6 @@ import ( type physicalTable struct { tableDTO internaldto.RelationDTO selectStmt sqlparser.SelectStatement - paramCollection internaldto.TableParameterCollection sqlSystem sql_system.SQLSystem underlyingSymbolTable symtab.SymTab } @@ -46,14 +44,6 @@ func (v *physicalTable) GetType() IndirectType { return PhysicalTableType } -func (v *physicalTable) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return v.paramCollection, v.paramCollection != nil -} - -func (v *physicalTable) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - v.paramCollection = paramCollection -} - func (v *physicalTable) GetUnderlyingSymTab() symtab.SymTab { return v.underlyingSymbolTable } @@ -86,14 +76,6 @@ func (v *physicalTable) GetRelationalColumnByIdentifier(name string) (typing.Rel return nil, false } -func (v *physicalTable) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (v *physicalTable) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (v *physicalTable) GetColumnByName(_ string) (typing.ColumnMetadata, bool) { return nil, false } diff --git a/internal/stackql/astindirect/select_indirect.go b/internal/stackql/astindirect/select_indirect.go index 290f5de10..dd1c32207 100644 --- a/internal/stackql/astindirect/select_indirect.go +++ b/internal/stackql/astindirect/select_indirect.go @@ -1,10 +1,8 @@ package astindirect import ( - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/symtab" "github.com/stackql/stackql/internal/stackql/typing" ) @@ -14,7 +12,6 @@ type parserSelectionIndirect struct { execObj *sqlparser.Exec insertObj *sqlparser.Insert selCtx drm.PreparedStatementCtx - paramCollection internaldto.TableParameterCollection underlyingSymbolTable symtab.SymTab } @@ -46,14 +43,6 @@ func (v *parserSelectionIndirect) GetType() IndirectType { return SelectType } -func (v *parserSelectionIndirect) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return v.paramCollection, v.paramCollection != nil -} - -func (v *parserSelectionIndirect) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - v.paramCollection = paramCollection -} - func (v *parserSelectionIndirect) GetRelationalColumns() []typing.RelationalColumn { return nil } @@ -78,14 +67,6 @@ func (v *parserSelectionIndirect) GetColumns() []typing.ColumnMetadata { return v.selCtx.GetNonControlColumns() } -func (v *parserSelectionIndirect) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (v *parserSelectionIndirect) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (v *parserSelectionIndirect) GetColumnByName(name string) (typing.ColumnMetadata, bool) { for _, col := range v.selCtx.GetNonControlColumns() { if col.GetIdentifier() == name { diff --git a/internal/stackql/astindirect/subquery_indirect.go b/internal/stackql/astindirect/subquery_indirect.go index 8802ad24c..cb01006b8 100644 --- a/internal/stackql/astindirect/subquery_indirect.go +++ b/internal/stackql/astindirect/subquery_indirect.go @@ -1,7 +1,6 @@ package astindirect import ( - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -14,7 +13,6 @@ type Subquery struct { subQuery *sqlparser.Subquery selectStmt sqlparser.SelectStatement selCtx drm.PreparedStatementCtx - paramCollection internaldto.TableParameterCollection underlyingSymbolTable symtab.SymTab } @@ -42,14 +40,6 @@ func (v *Subquery) GetType() IndirectType { return SubqueryType } -func (v *Subquery) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return v.paramCollection, v.paramCollection != nil -} - -func (v *Subquery) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - v.paramCollection = paramCollection -} - func (v *Subquery) GetRelationalColumns() []typing.RelationalColumn { return nil } @@ -78,14 +68,6 @@ func (v *Subquery) GetColumns() []typing.ColumnMetadata { return v.selCtx.GetNonControlColumns() } -func (v *Subquery) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (v *Subquery) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (v *Subquery) GetColumnByName(name string) (typing.ColumnMetadata, bool) { for _, col := range v.selCtx.GetNonControlColumns() { if col.GetIdentifier() == name { diff --git a/internal/stackql/astindirect/view_indirect.go b/internal/stackql/astindirect/view_indirect.go index 830abafed..9c0d757ab 100644 --- a/internal/stackql/astindirect/view_indirect.go +++ b/internal/stackql/astindirect/view_indirect.go @@ -3,7 +3,6 @@ package astindirect import ( "fmt" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -16,7 +15,6 @@ type View struct { viewDTO internaldto.RelationDTO selectStmt sqlparser.SelectStatement selCtx drm.PreparedStatementCtx - paramCollection internaldto.TableParameterCollection underlyingSymbolTable symtab.SymTab next Indirect isElide bool @@ -54,14 +52,6 @@ func (v *View) GetRelationalColumns() []typing.RelationalColumn { return nil } -func (v *View) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return v.paramCollection, v.paramCollection != nil -} - -func (v *View) SetAssignedParameters(paramCollection internaldto.TableParameterCollection) { - v.paramCollection = paramCollection -} - func (v *View) GetUnderlyingSymTab() symtab.SymTab { return v.underlyingSymbolTable } @@ -78,14 +68,6 @@ func (v *View) GetColumns() []typing.ColumnMetadata { return v.selCtx.GetNonControlColumns() } -func (v *View) GetOptionalParameters() map[string]formulation.Addressable { - return nil -} - -func (v *View) GetRequiredParameters() map[string]formulation.Addressable { - return nil -} - func (v *View) GetColumnByName(name string) (typing.ColumnMetadata, bool) { nccs := v.selCtx.GetNonControlColumns() for _, col := range nccs { diff --git a/internal/stackql/astvisit/assign_leftover_references.go b/internal/stackql/astvisit/assign_leftover_references.go deleted file mode 100644 index 800b4e567..000000000 --- a/internal/stackql/astvisit/assign_leftover_references.go +++ /dev/null @@ -1,783 +0,0 @@ -//nolint:dupl,cyclop // defer uplifts on analysers -package astvisit - -import ( - "fmt" - "strings" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/taxonomy" -) - -var ( - _ LeftoverReferencesAstVisitor = &standardLeftoverReferencesAstVisitor{} -) - -// TODO: must be view-aware. -type LeftoverReferencesAstVisitor interface { - sqlparser.SQLAstVisitor - GetTablesFoundThisIteration() map[sqlparser.TableExpr]taxonomy.AnnotationCtx -} - -type standardLeftoverReferencesAstVisitor struct { - colRefs parserutil.ColTableMap - tableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx - thisIterationTableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx - annotatedAST annotatedast.AnnotatedAst -} - -func NewLeftoverReferencesAstVisitor( - annotatedAST annotatedast.AnnotatedAst, - colRefs parserutil.ColTableMap, - tableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx, -) LeftoverReferencesAstVisitor { - copyColRefs := make(parserutil.ColTableMap) - for k, v := range colRefs { - copyColRefs[k] = v - } - return &standardLeftoverReferencesAstVisitor{ - annotatedAST: annotatedAST, - colRefs: copyColRefs, - tableToAnnotationCtx: tableToAnnotationCtx, - thisIterationTableToAnnotationCtx: make(map[sqlparser.TableExpr]taxonomy.AnnotationCtx), - } -} - -//nolint:lll // defer analyser uplifts -func (v *standardLeftoverReferencesAstVisitor) GetTablesFoundThisIteration() map[sqlparser.TableExpr]taxonomy.AnnotationCtx { - return v.thisIterationTableToAnnotationCtx -} - -//nolint:govet,ineffassign,staticcheck // defer analyser uplifts -func (v *standardLeftoverReferencesAstVisitor) findTableLeftover( - colName *sqlparser.ColName, -) (sqlparser.TableExpr, error) { - for tb, md := range v.tableToAnnotationCtx { - // TODO: make recursive for views - tm := md.GetTableMeta() - if tm == nil { - return nil, fmt.Errorf("cannot accomodate nil tables") - } - selectable, _, err := tm.GetSelectSchemaAndObjectPath() - if err != nil { - return nil, err - } - properties, err := selectable.GetProperties() //nolint:wastedassign // TODO: review - for k := range properties { - if k == colName.GetRawVal() { - ref, err := parserutil.NewUnknownTypeColumnarReference(k) - if err != nil { - return nil, err - } - v.colRefs[ref] = tb - return tb, nil - } - } - } - return nil, fmt.Errorf("could not locate table corresponding to expression '%s'", colName.GetRawVal()) -} - -//nolint:dupl,funlen,gocognit,gocyclo,cyclop,errcheck,staticcheck,gocritic,lll,nestif,govet,gomnd,exhaustive,revive // defer uplifts on analysers -func (v *standardLeftoverReferencesAstVisitor) Visit(node sqlparser.SQLNode) error { - var err error - - switch node := node.(type) { - case *sqlparser.Select: - var options string - addIf := func(b bool, s string) { - if b { - options += s - } - } - addIf(node.Distinct, sqlparser.DistinctStr) - if node.Cache != nil { - if *node.Cache { - options += sqlparser.SQLCacheStr - } else { - options += sqlparser.SQLNoCacheStr - } - } - addIf(node.StraightJoinHint, sqlparser.StraightJoinHint) - addIf(node.SQLCalcFoundRows, sqlparser.SQLCalcFoundRowsStr) - - //nolint:errcheck // defer uplifts on analysers - if node.Comments != nil { - node.Comments.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.SelectExprs != nil { - node.SelectExprs.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.From != nil { - node.From.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.Where != nil { - node.Where.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.GroupBy != nil { - node.GroupBy.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.Having != nil { - node.Having.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.OrderBy != nil { - node.OrderBy.Accept(v) - } - //nolint:errcheck // defer uplifts on analysers - if node.Limit != nil { - node.Limit.Accept(v) - } - return nil - - case *sqlparser.ParenSelect: - node.Accept(v) - - case *sqlparser.Auth: - return nil - - case *sqlparser.AuthRevoke: - return nil - case *sqlparser.Sleep: - return nil - - case *sqlparser.Union: - err = node.FirstStatement.Accept(v) - if err != nil { - return err - } - for _, us := range node.UnionSelects { - err = us.Accept(v) - if err != nil { - return err - } - } - return nil - - case *sqlparser.UnionSelect: - return node.Statement.Accept(v) - - case *sqlparser.Stream: - err = node.Comments.Accept(v) - if err != nil { - return err - } - err = node.SelectExpr.Accept(v) - if err != nil { - return err - } - return node.Table.Accept(v) - - case *sqlparser.Insert: - - case *sqlparser.Update: - - case *sqlparser.Delete: - - case *sqlparser.Set: - - case *sqlparser.SetTransaction: - - case *sqlparser.DBDDL: - - case *sqlparser.DDL: - switch node.Action { - case sqlparser.CreateStr: - case sqlparser.DropStr: - case sqlparser.RenameStr: - case sqlparser.AlterStr: - case sqlparser.FlushStr: - case sqlparser.CreateVindexStr: - case sqlparser.DropVindexStr: - case sqlparser.AddVschemaTableStr: - case sqlparser.DropVschemaTableStr: - case sqlparser.AddColVindexStr: - case sqlparser.DropColVindexStr: - case sqlparser.AddSequenceStr: - case sqlparser.AddAutoIncStr: - default: - } - - case *sqlparser.OptLike: - - case *sqlparser.PartitionSpec: - switch node.Action { - case sqlparser.ReorganizeStr: - default: - } - - case *sqlparser.PartitionDefinition: - - case *sqlparser.TableSpec: - - case *sqlparser.ColumnDefinition: - - // Format returns a canonical string representation of the type and all relevant options - case *sqlparser.ColumnType: - ct := node - - if ct.Length != nil && ct.Scale != nil { - - } else if ct.Length != nil { - } - - if ct.EnumValues != nil { - } - - opts := make([]string, 0, 16) //nolint:mnd // TODO: tech debt sweep mnd hacks - if ct.Unsigned { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNSIGNED]) - } - if ct.Zerofill { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ZEROFILL]) - } - if ct.Charset != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.CHARACTER], sqlparser.KeywordStrings[sqlparser.SET], ct.Charset) - } - if ct.Collate != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COLLATE], ct.Collate) - } - if ct.NotNull { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.NOT], sqlparser.KeywordStrings[sqlparser.NULL]) - } - if ct.Default != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.DEFAULT], sqlparser.String(ct.Default)) - } - if ct.OnUpdate != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ON], sqlparser.KeywordStrings[sqlparser.UPDATE], sqlparser.String(ct.OnUpdate)) - } - if ct.Autoincrement { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.AUTO_INCREMENT]) - } - if ct.Comment != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COMMENT_KEYWORD], sqlparser.String(ct.Comment)) - } - if ct.KeyOpt == sqlparser.ColKeyPrimary { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.PRIMARY], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeyUnique { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE]) - } - if ct.KeyOpt == sqlparser.ColKeyUniqueKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeySpatialKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.SPATIAL], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.KEY]) - } - - if len(opts) != 0 { - } - - case *sqlparser.IndexDefinition: - idx := node - for i, col := range idx.Columns { - if i != 0 { - } else { - } - if col.Length != nil { - } - } - - for _, opt := range idx.Options { - if opt.Using != "" { - } else { - } - } - - case *sqlparser.IndexInfo: - ii := node - if ii.Primary { - } else { - if !ii.Name.IsEmpty() { - } - } - - case *sqlparser.AutoIncSpec: - - case *sqlparser.VindexSpec: - - numParams := len(node.Params) - if numParams != 0 { - for i, p := range node.Params { - logging.GetLogger().Debugf("%v\n", p) - if i != 0 { - } - } - } - - case sqlparser.VindexParam: - - case *sqlparser.ConstraintDefinition: - c := node - if c.Name != "" { - } - - case sqlparser.ReferenceAction: - a := node - switch a { - case sqlparser.Restrict: - case sqlparser.Cascade: - case sqlparser.NoAction: - case sqlparser.SetNull: - case sqlparser.SetDefault: - } - - case *sqlparser.ForeignKeyDefinition: - f := node - if f.OnDelete != sqlparser.DefaultAction { - } - if f.OnUpdate != sqlparser.DefaultAction { - } - - case *sqlparser.Show: - nodeType := strings.ToLower(node.Type) - if (nodeType == "tables" || nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.ShowTablesOpt != nil { - opt := node.ShowTablesOpt - if node.Extended != "" { - } else { - } - if (nodeType == "columns" || nodeType == "fields") && node.HasOnTable() { - } - if (nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.HasOnTable() { - } - if opt.DbName != "" { - } - return nil - } - if node.Scope == "" { - } else { - } - if node.HasOnTable() { - } - if nodeType == "collation" && node.ShowCollationFilterOpt != nil { - } - if nodeType == "charset" && node.ShowTablesOpt != nil { - } - if node.HasTable() { - } - - case *sqlparser.ShowFilter: - if node == nil { - return nil - } - if node.Like != "" { - } else { - } - - case *sqlparser.Use: - if node.DBName.GetRawVal() != "" { - } else { - } - - case *sqlparser.Commit: - - case *sqlparser.Begin: - - case *sqlparser.Rollback: - - case *sqlparser.SRollback: - - case *sqlparser.Savepoint: - - case *sqlparser.Release: - - case *sqlparser.Explain: - switch node.Type { - case "": // do nothing - case sqlparser.AnalyzeStr: - default: - } - - case *sqlparser.OtherRead: - - case *sqlparser.DescribeTable: - - case *sqlparser.DescribeMethod: - - case *sqlparser.OtherAdmin: - - case sqlparser.Comments: - - case sqlparser.SelectExprs: - for _, n := range node { - err = v.Visit(n) - if err != nil { - return err - } - } - - case *sqlparser.StarExpr: - if !node.TableName.IsEmpty() { - } - - case *sqlparser.AliasedExpr: - err = v.Visit(node.Expr) - if err != nil { - return err - } - if !node.As.IsEmpty() { - } - - case sqlparser.Nextval: - - case sqlparser.Columns: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.Partitions: - if node == nil { - return nil - } - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.TableExprs: - for _, n := range node { - n.Accept(v) - } - - case *sqlparser.AliasedTableExpr: - aliasStr := node.As.GetRawVal() - if aliasStr != "" { - // v.aliasMap[aliasStr] = node - } - if node.Expr != nil { - node.Expr.Accept(v) - } - if node.Partitions != nil { - node.Partitions.Accept(v) - } - if !node.As.IsEmpty() { - node.As.Accept(v) - } - if node.Hints != nil { - node.Hints.Accept(v) - } - - case sqlparser.TableNames: - for _, n := range node { - n.Accept(v) - } - - case sqlparser.TableName: - if node.IsEmpty() { - return nil - } - if !node.QualifierThird.IsEmpty() { - } - if !node.QualifierSecond.IsEmpty() { - } - if !node.Qualifier.IsEmpty() { - } - - case *sqlparser.ParenTableExpr: - - case sqlparser.JoinCondition: - if node.On != nil { - err := node.On.Accept(v) - if err != nil { - return err - } - } - if node.Using != nil { - } - - case *sqlparser.JoinTableExpr: - err := node.LeftExpr.Accept(v) - if err != nil { - return err - } - err = node.RightExpr.Accept(v) - if err != nil { - return err - } - err = node.Condition.Accept(v) - if err != nil { - return err - } - - case *sqlparser.IndexHints: - if len(node.Indexes) == 0 { - } else { - for _, n := range node.Indexes { - logging.GetLogger().Debugf("%v\n", n) - } - } - - case *sqlparser.Where: - if node == nil || node.Expr == nil { - return nil - } - return node.Expr.Accept(v) - - case sqlparser.Exprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.AndExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.OrExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.XorExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.NotExpr: - return node.Expr.Accept(v) - - case *sqlparser.ComparisonExpr: - err := node.Left.Accept(v) - if err != nil { - return err - } - err = node.Right.Accept(v) - if err != nil { - return err - } - // switch lt := node.Left.(type) { - // case *sqlparser.ColName: - // switch rt := node.Right.(type) { - // case *sqlparser.SQLVal: - // v.tables[lt] = rt - // default: - // } - // default: - // switch rt := node.Right.(type) { - // case *sqlparser.SQLVal: - // case *sqlparser.ColName: - // // v.tables[rt] = lt - // default: - // } - // } - return nil - - case *sqlparser.RangeCond: - - case *sqlparser.IsExpr: - return node.Expr.Accept(v) - - case *sqlparser.ExistsExpr: - return nil - - case *sqlparser.SQLVal: - switch node.Type { - case sqlparser.StrVal: - case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.HexNum: - case sqlparser.HexVal: - case sqlparser.BitVal: - case sqlparser.ValArg: - default: - } - - case *sqlparser.NullVal: - - case sqlparser.BoolVal: - if node { - } else { - } - - case *sqlparser.ColName: - k, err := parserutil.NewUnknownTypeColumnarReference(node) - if err != nil { - return err - } - t, ok := v.colRefs[k] - if ok { - annCtx, ok := v.tableToAnnotationCtx[t] - if !ok { - return fmt.Errorf("found orphan table for col '%s'", node.GetRawVal()) - } - v.thisIterationTableToAnnotationCtx[t] = annCtx - return nil - } - t, err = v.findTableLeftover(node) - if err != nil { - return err - } - annCtx, ok := v.tableToAnnotationCtx[t] - if !ok { - return fmt.Errorf("found orphan table for col '%s'", node.GetRawVal()) - } - v.thisIterationTableToAnnotationCtx[t] = annCtx - return nil - - case sqlparser.ValTuple: - - case *sqlparser.Subquery: - - case sqlparser.ListArg: - - case *sqlparser.BinaryExpr: - - case *sqlparser.UnaryExpr: - if _, unary := node.Expr.(*sqlparser.UnaryExpr); unary { - // They have same precedence so parenthesis is not required. - return nil - } - - case *sqlparser.IntervalExpr: - - case *sqlparser.TimestampFuncExpr: - - case *sqlparser.CurTimeFuncExpr: - - case *sqlparser.CollateExpr: - - case *sqlparser.FuncExpr: - err := node.Exprs.Accept(v) - if err != nil { - return err - } - if node.Distinct { - } - if !node.Qualifier.IsEmpty() { - } - // Function names should not be back-quoted even - // if they match a reserved word, only if they contain illegal characters - funcName := node.Name.String() - - if sqlparser.ContainEscapableChars(funcName, sqlparser.NoAt) { - } else { - } - - case *sqlparser.GroupConcatExpr: - - case *sqlparser.ValuesFuncExpr: - - case *sqlparser.SubstrExpr: - if node.Name != nil { - } else { - } - - if node.To == nil { - } else { - } - - case *sqlparser.ConvertExpr: - - case *sqlparser.ConvertUsingExpr: - - case *sqlparser.ConvertType: - if node.Length != nil { - if node.Scale != nil { - } - } - if node.Charset != "" { - } - - case *sqlparser.MatchExpr: - - case *sqlparser.CaseExpr: - if node.Expr != nil { - } - for _, when := range node.Whens { - logging.GetLogger().Debugf("%v\n", when) - } - if node.Else != nil { - } - - case *sqlparser.Default: - if node.ColName != "" { - } - - case *sqlparser.When: - - case sqlparser.GroupBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.OrderBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.Order: - if node, ok := node.Expr.(*sqlparser.NullVal); ok { - logging.GetLogger().Debugf("%v\n", node) - return nil - } - if node, ok := node.Expr.(*sqlparser.FuncExpr); ok { - if node.Name.Lowered() == "rand" { - return nil - } - } - - case *sqlparser.Limit: - if node == nil { - return nil - } - if node.Offset != nil { - } - - case sqlparser.Values: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.UpdateExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.UpdateExpr: - - case sqlparser.SetExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.SetExpr: - if node.Scope != "" { - } - // We don't have to backtick set variable names. - switch { - case node.Name.EqualString("charset") || node.Name.EqualString("names"): - case node.Name.EqualString(sqlparser.TransactionStr): - default: - } - - case sqlparser.OnDup: - if node == nil { - return nil - } - - case sqlparser.ColIdent: - for i := sqlparser.NoAt; i < node.GetAtCount(); i++ { - } - - case sqlparser.TableIdent: - - case *sqlparser.IsolationLevel: - - case *sqlparser.AccessMode: - } - return nil -} diff --git a/internal/stackql/astvisit/from_rewrite.go b/internal/stackql/astvisit/from_rewrite.go deleted file mode 100644 index 80050c529..000000000 --- a/internal/stackql/astvisit/from_rewrite.go +++ /dev/null @@ -1,1222 +0,0 @@ -package astvisit - -import ( - "fmt" - "regexp" - "strings" - - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/astindirect" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/sql_system" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/pkg/astformat" - "github.com/stackql/stackql/pkg/textutil" - - "github.com/stackql/stackql-parser/go/sqltypes" - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -var ( - _ FromRewriteAstVisitor = &standardFromRewriteAstVisitor{} -) - -// The parser folds CROSS JOIN into sqlparser.JoinStr, so it is re-emitted here. -const crossJoinStr = "cross join" - -type FromRewriteAstVisitor interface { - sqlparser.SQLAstVisitor - GetIndirectContexts() []drm.PreparedStatementCtx - GetRewrittenQuery() string - SetAvoidSQLSourceNaming(bool) - GetHoistedOnClauseTables() []sqlparser.SQLNode -} - -// Need not be view-aware. -type standardFromRewriteAstVisitor struct { - iDColumnName string - rewrittenQuery string - shouldCollectTables bool - namespaceCollection tablenamespace.Collection - sqlSystem sql_system.SQLSystem - formatter sqlparser.NodeFormatter - annotations taxonomy.AnnotationCtxMap - dc drm.Config - annotatedAST annotatedast.AnnotatedAst - indirectContexts []drm.PreparedStatementCtx - isAvoidSQLSourceNaming bool - hoistedOnClauseTables []sqlparser.SQLNode -} - -func NewFromRewriteAstVisitor( - annotatedAST annotatedast.AnnotatedAst, - iDColumnName string, - shouldCollectTables bool, - sqlSystem sql_system.SQLSystem, - formatter sqlparser.NodeFormatter, - namespaceCollection tablenamespace.Collection, - annotations taxonomy.AnnotationCtxMap, - dc drm.Config, -) FromRewriteAstVisitor { - return &standardFromRewriteAstVisitor{ - annotatedAST: annotatedAST, - iDColumnName: iDColumnName, - shouldCollectTables: shouldCollectTables, - namespaceCollection: namespaceCollection, - sqlSystem: sqlSystem, - formatter: formatter, - annotations: annotations, - dc: dc, - } -} - -func (v *standardFromRewriteAstVisitor) GetHoistedOnClauseTables() []sqlparser.SQLNode { - return v.hoistedOnClauseTables -} - -func (v *standardFromRewriteAstVisitor) SetAvoidSQLSourceNaming(isAvoidSQLSourceNaming bool) { - v.isAvoidSQLSourceNaming = isAvoidSQLSourceNaming -} - -func (v *standardFromRewriteAstVisitor) GetRewrittenQuery() string { - return v.rewrittenQuery -} - -func (v *standardFromRewriteAstVisitor) GetIndirectContexts() []drm.PreparedStatementCtx { - return v.indirectContexts -} - -//nolint:dupl,funlen,gocognit,gocyclo,cyclop,errcheck,gocritic,lll,exhaustive,nestif,gomnd // defer uplifts on analysers -func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { - buf := sqlparser.NewTrackedBuffer(v.formatter) - - switch node := node.(type) { - case *sqlparser.Select: - return fmt.Errorf("unsupported subquery pattern") - - case *sqlparser.ParenSelect: - node.Accept(v) - selStr := v.GetRewrittenQuery() - rq := fmt.Sprintf("(%v)", selStr) - v.rewrittenQuery = rq - - case *sqlparser.Auth: - var stackqlOpt string - if node.SessionAuth { - stackqlOpt = "stackql " - } - rq := fmt.Sprintf("%sAUTH %v %s %v %v", stackqlOpt, node.Provider, node.Type, node.KeyFilePath, node.KeyEnvVar) - v.rewrittenQuery = rq - - case *sqlparser.AuthRevoke: - var stackqlOpt string - if node.SessionAuth { - stackqlOpt = "stackql " - } - rq := fmt.Sprintf("%sauth revoke %v", stackqlOpt, node.Provider) - v.rewrittenQuery = rq - - case *sqlparser.Sleep: - rq := fmt.Sprintf("sleep %v", node.Duration) - v.rewrittenQuery = rq - - case *sqlparser.Union: - buf.AstPrintf(node, "%v", node.FirstStatement) - for _, us := range node.UnionSelects { - buf.AstPrintf(node, "%v", us) - } - buf.AstPrintf(node, "%v%v%s", node.OrderBy, node.Limit, node.Lock) - v.rewrittenQuery = buf.String() - - case *sqlparser.UnionSelect: - buf.AstPrintf(node, " %s %v", node.Type, node.Statement) - v.rewrittenQuery = buf.String() - - case *sqlparser.Stream: - buf.AstPrintf(node, "stream %v%v from %v", - node.Comments, node.SelectExpr, node.Table) - v.rewrittenQuery = buf.String() - - case *sqlparser.Insert: - buf.AstPrintf(node, "%s %v%sinto %v%v%v %v%v", - node.Action, - node.Comments, node.Ignore, - node.Table, node.Partitions, node.Columns, node.Rows, node.OnDup) - v.rewrittenQuery = buf.String() - - case *sqlparser.Update: - buf.AstPrintf(node, "update %v%s%v set %v%v%v%v", - node.Comments, node.Ignore, node.TableExprs, - node.Exprs, node.Where, node.OrderBy, node.Limit) - v.rewrittenQuery = buf.String() - - case *sqlparser.Delete: - buf.AstPrintf(node, "delete %v", node.Comments) - if node.Targets != nil { - buf.AstPrintf(node, "%v ", node.Targets) - } - buf.AstPrintf(node, "from %v%v%v%v%v", node.TableExprs, node.Partitions, node.Where, node.OrderBy, node.Limit) - v.rewrittenQuery = buf.String() - - case *sqlparser.Set: - buf.AstPrintf(node, "set %v%v", node.Comments, node.Exprs) - v.rewrittenQuery = buf.String() - - case *sqlparser.SetTransaction: - if node.Scope == "" { - buf.AstPrintf(node, "set %vtransaction ", node.Comments) - } else { - buf.AstPrintf(node, "set %v%s transaction ", node.Comments, node.Scope) - } - - for i, char := range node.Characteristics { - if i > 0 { - buf.WriteString(", ") - } - buf.AstPrintf(node, "%v", char) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.DBDDL: - switch node.Action { - case sqlparser.CreateStr, sqlparser.AlterStr: - buf.WriteString(fmt.Sprintf("%s database %s", node.Action, node.DBName)) - case sqlparser.DropStr: - exists := "" - if node.IfExists { - exists = " if exists" - } - buf.WriteString(fmt.Sprintf("%s database%s %v", node.Action, exists, node.DBName)) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.DDL: - switch node.Action { - case sqlparser.CreateStr: - if node.OptLike != nil { - buf.AstPrintf(node, "%s table %v %v", node.Action, node.Table, node.OptLike) - } else if node.TableSpec != nil { - buf.AstPrintf(node, "%s table %v %v", node.Action, node.Table, node.TableSpec) - } else { - buf.AstPrintf(node, "%s table %v", node.Action, node.Table) - } - case sqlparser.DropStr: - exists := "" - if node.IfExists { - exists = " if exists" - } - buf.AstPrintf(node, "%s table%s %v", node.Action, exists, node.FromTables) - case sqlparser.RenameStr: - buf.AstPrintf(node, "%s table %v to %v", node.Action, node.FromTables[0], node.ToTables[0]) - for i := 1; i < len(node.FromTables); i++ { - buf.AstPrintf(node, ", %v to %v", node.FromTables[i], node.ToTables[i]) - } - case sqlparser.AlterStr: - if node.PartitionSpec != nil { - buf.AstPrintf(node, "%s table %v %v", node.Action, node.Table, node.PartitionSpec) - } else { - buf.AstPrintf(node, "%s table %v", node.Action, node.Table) - } - case sqlparser.FlushStr: - buf.AstPrintf(node, "%s", node.Action) - case sqlparser.CreateVindexStr: - buf.AstPrintf(node, "alter vschema create vindex %v %v", node.Table, node.VindexSpec) - case sqlparser.DropVindexStr: - buf.AstPrintf(node, "alter vschema drop vindex %v", node.Table) - case sqlparser.AddVschemaTableStr: - buf.AstPrintf(node, "alter vschema add table %v", node.Table) - case sqlparser.DropVschemaTableStr: - buf.AstPrintf(node, "alter vschema drop table %v", node.Table) - case sqlparser.AddColVindexStr: - buf.AstPrintf(node, "alter vschema on %v add vindex %v (", node.Table, node.VindexSpec.Name) - for i, col := range node.VindexCols { - if i != 0 { - buf.AstPrintf(node, ", %v", col) - } else { - buf.AstPrintf(node, "%v", col) - } - } - buf.AstPrintf(node, ")") - if node.VindexSpec.Type.String() != "" { - buf.AstPrintf(node, " %v", node.VindexSpec) - } - case sqlparser.DropColVindexStr: - buf.AstPrintf(node, "alter vschema on %v drop vindex %v", node.Table, node.VindexSpec.Name) - case sqlparser.AddSequenceStr: - buf.AstPrintf(node, "alter vschema add sequence %v", node.Table) - case sqlparser.AddAutoIncStr: - buf.AstPrintf(node, "alter vschema on %v add auto_increment %v", node.Table, node.AutoIncSpec) - default: - buf.AstPrintf(node, "%s table %v", node.Action, node.Table) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.OptLike: - buf.AstPrintf(node, "like %v", node.LikeTable) - v.rewrittenQuery = buf.String() - - case *sqlparser.PartitionSpec: - switch node.Action { - case sqlparser.ReorganizeStr: - buf.AstPrintf(node, "%s %v into (", node.Action, node.Name) - var prefix string - for _, pd := range node.Definitions { - buf.AstPrintf(node, "%s%v", prefix, pd) - prefix = ", " - } - buf.AstPrintf(node, ")") - default: - panic("unimplemented") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.PartitionDefinition: - if !node.Maxvalue { - buf.AstPrintf(node, "partition %v values less than (%v)", node.Name, node.Limit) - } else { - buf.AstPrintf(node, "partition %v values less than (maxvalue)", node.Name) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.TableSpec: - ts := node - buf.AstPrintf(ts, "(\n") - for i, col := range ts.Columns { - if i == 0 { - buf.AstPrintf(ts, "\t%v", col) - } else { - buf.AstPrintf(ts, ",\n\t%v", col) - } - } - for _, idx := range ts.Indexes { - buf.AstPrintf(ts, ",\n\t%v", idx) - } - for _, c := range ts.Constraints { - buf.AstPrintf(ts, ",\n\t%v", c) - } - - buf.AstPrintf(ts, "\n)%s", strings.Replace(ts.Options, ", ", ",\n ", -1)) - v.rewrittenQuery = buf.String() - - case *sqlparser.ColumnDefinition: - col := node - buf.AstPrintf(col, "%v %v", col.Name, &col.Type) - v.rewrittenQuery = buf.String() - - // Format returns a canonical string representation of the type and all relevant options - case *sqlparser.ColumnType: - ct := node - buf.AstPrintf(ct, "%s", ct.Type) - - if ct.Length != nil && ct.Scale != nil { - buf.AstPrintf(ct, "(%v,%v)", ct.Length, ct.Scale) - } else if ct.Length != nil { - buf.AstPrintf(ct, "(%v)", ct.Length) - } - - if ct.EnumValues != nil { - buf.AstPrintf(ct, "(%s)", strings.Join(ct.EnumValues, ", ")) - } - - opts := make([]string, 0, 16) //nolint:mnd // TODO: tech debt sweep mnd hacks - if ct.Unsigned { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNSIGNED]) - } - if ct.Zerofill { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ZEROFILL]) - } - if ct.Charset != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.CHARACTER], sqlparser.KeywordStrings[sqlparser.SET], ct.Charset) - } - if ct.Collate != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COLLATE], ct.Collate) - } - if ct.NotNull { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.NOT], sqlparser.KeywordStrings[sqlparser.NULL]) - } - if ct.Default != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.DEFAULT], sqlparser.String(ct.Default)) - } - if ct.OnUpdate != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ON], sqlparser.KeywordStrings[sqlparser.UPDATE], sqlparser.String(ct.OnUpdate)) - } - if ct.Autoincrement { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.AUTO_INCREMENT]) - } - if ct.Comment != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COMMENT_KEYWORD], sqlparser.String(ct.Comment)) - } - if ct.KeyOpt == sqlparser.ColKeyPrimary { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.PRIMARY], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeyUnique { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE]) - } - if ct.KeyOpt == sqlparser.ColKeyUniqueKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeySpatialKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.SPATIAL], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.KEY]) - } - - if len(opts) != 0 { - buf.AstPrintf(ct, " %s", strings.Join(opts, " ")) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.IndexDefinition: - idx := node - buf.AstPrintf(idx, "%v (", idx.Info) - for i, col := range idx.Columns { - if i != 0 { - buf.AstPrintf(idx, ", %v", col.Column) - } else { - buf.AstPrintf(idx, "%v", col.Column) - } - if col.Length != nil { - buf.AstPrintf(idx, "(%v)", col.Length) - } - } - buf.AstPrintf(idx, ")") - - for _, opt := range idx.Options { - buf.AstPrintf(idx, " %s", opt.Name) - if opt.Using != "" { - buf.AstPrintf(idx, " %s", opt.Using) - } else { - buf.AstPrintf(idx, " %v", opt.Value) - } - } - v.rewrittenQuery = buf.String() - - case *sqlparser.IndexInfo: - ii := node - if ii.Primary { - buf.AstPrintf(ii, "%s", ii.Type) - } else { - buf.AstPrintf(ii, "%s", ii.Type) - if !ii.Name.IsEmpty() { - buf.AstPrintf(ii, " %v", ii.Name) - } - } - v.rewrittenQuery = buf.String() - - case *sqlparser.AutoIncSpec: - buf.AstPrintf(node, "%v ", node.Column) - buf.AstPrintf(node, "using %v", node.Sequence) - v.rewrittenQuery = buf.String() - - case *sqlparser.VindexSpec: - buf.AstPrintf(node, "using %v", node.Type) - - numParams := len(node.Params) - if numParams != 0 { - buf.AstPrintf(node, " with ") - for i, p := range node.Params { - if i != 0 { - buf.AstPrintf(node, ", ") - } - buf.AstPrintf(node, "%v", p) - } - } - v.rewrittenQuery = buf.String() - - case sqlparser.VindexParam: - buf.AstPrintf(node, "%s=%s", node.Key.String(), node.Val) - v.rewrittenQuery = buf.String() - - case *sqlparser.ConstraintDefinition: - c := node - if c.Name != "" { - buf.AstPrintf(c, "constraint %s ", c.Name) - } - c.Details.Format(buf) - v.rewrittenQuery = buf.String() - - case sqlparser.ReferenceAction: - a := node - switch a { - case sqlparser.Restrict: - buf.WriteString("restrict") - case sqlparser.Cascade: - buf.WriteString("cascade") - case sqlparser.NoAction: - buf.WriteString("no action") - case sqlparser.SetNull: - buf.WriteString("set null") - case sqlparser.SetDefault: - buf.WriteString("set default") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.ForeignKeyDefinition: - f := node - buf.AstPrintf(f, "foreign key %v references %v %v", f.Source, f.ReferencedTable, f.ReferencedColumns) - if f.OnDelete != sqlparser.DefaultAction { - buf.AstPrintf(f, " on delete %v", f.OnDelete) - } - if f.OnUpdate != sqlparser.DefaultAction { - buf.AstPrintf(f, " on update %v", f.OnUpdate) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.Show: - nodeType := strings.ToLower(node.Type) - if (nodeType == "tables" || nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.ShowTablesOpt != nil { - opt := node.ShowTablesOpt - if node.Extended != "" { - buf.AstPrintf(node, "show %s%s", node.Extended, nodeType) - } else { - buf.AstPrintf(node, "show %s%s", opt.Full, nodeType) - } - if (nodeType == "columns" || nodeType == "fields") && node.HasOnTable() { - buf.AstPrintf(node, " from %v", node.OnTable) - } - if (nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.HasOnTable() { - buf.AstPrintf(node, " from %v", node.OnTable) - } - if opt.DbName != "" { - buf.AstPrintf(node, " from %s", opt.DbName) - } - buf.AstPrintf(node, "%v", opt.Filter) - return nil - } - if node.Scope == "" { - buf.AstPrintf(node, "show %s", nodeType) - } else { - buf.AstPrintf(node, "show %s %s", node.Scope, nodeType) - } - if node.HasOnTable() { - buf.AstPrintf(node, " on %v", node.OnTable) - } - if nodeType == "collation" && node.ShowCollationFilterOpt != nil { - buf.AstPrintf(node, " where %v", node.ShowCollationFilterOpt) - } - if nodeType == "charset" && node.ShowTablesOpt != nil { - buf.AstPrintf(node, "%v", node.ShowTablesOpt.Filter) - } - if node.HasTable() { - buf.AstPrintf(node, " %v", node.Table) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.ShowFilter: - if node == nil { - return nil - } - if node.Like != "" { - buf.AstPrintf(node, " like '%s'", node.Like) - } else { - buf.AstPrintf(node, " where %v", node.Filter) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.Use: - if node.DBName.GetRawVal() != "" { - buf.AstPrintf(node, "use %v", node.DBName) - } else { - buf.AstPrintf(node, "use") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.Commit: - buf.WriteString("commit") - v.rewrittenQuery = buf.String() - - case *sqlparser.Begin: - buf.WriteString("begin") - v.rewrittenQuery = buf.String() - - case *sqlparser.Rollback: - buf.WriteString("rollback") - v.rewrittenQuery = buf.String() - - case *sqlparser.SRollback: - buf.AstPrintf(node, "rollback to %v", node.Name) - v.rewrittenQuery = buf.String() - - case *sqlparser.Savepoint: - buf.AstPrintf(node, "savepoint %v", node.Name) - v.rewrittenQuery = buf.String() - - case *sqlparser.Release: - buf.AstPrintf(node, "release savepoint %v", node.Name) - v.rewrittenQuery = buf.String() - - case *sqlparser.Explain: - format := "" - switch node.Type { - case "": // do nothing - case sqlparser.AnalyzeStr: - format = sqlparser.AnalyzeStr + " " - default: - format = "format = " + node.Type + " " - } - buf.AstPrintf(node, "explain %s%v", format, node.Statement) - v.rewrittenQuery = buf.String() - - case *sqlparser.OtherRead: - buf.WriteString("otherread") - v.rewrittenQuery = buf.String() - - case *sqlparser.DescribeTable: - buf.WriteString("describetable") - v.rewrittenQuery = buf.String() - - case *sqlparser.DescribeMethod: - buf.WriteString("describemethod") - v.rewrittenQuery = buf.String() - - case *sqlparser.OtherAdmin: - buf.WriteString("otheradmin") - v.rewrittenQuery = buf.String() - - case sqlparser.Comments: - for _, c := range node { - buf.AstPrintf(node, "%s ", c) - } - v.rewrittenQuery = buf.String() - - case sqlparser.SelectExprs: - var prefix string - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case *sqlparser.StarExpr: - if !node.TableName.IsEmpty() { - buf.AstPrintf(node, "%v.", node.TableName) - } - buf.AstPrintf(node, "*") - v.rewrittenQuery = buf.String() - - case *sqlparser.AliasedExpr: - buf.AstPrintf(node, "%v", node.Expr) - if !node.As.IsEmpty() { - buf.AstPrintf(node, " as %v", node.As) - } - v.rewrittenQuery = buf.String() - - case sqlparser.Nextval: - buf.AstPrintf(node, "next %v values", node.Expr) - v.rewrittenQuery = buf.String() - - case sqlparser.Columns: - if node == nil { - return nil - } - prefix := "(" - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - buf.WriteString(")") - v.rewrittenQuery = buf.String() - - case sqlparser.Partitions: - if node == nil { - return nil - } - prefix := " partition (" - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - buf.WriteString(")") - v.rewrittenQuery = buf.String() - - case *sqlparser.TableValuedFuncTableExpr: - buf.AstPrintf(node, "%v", node) - v.rewrittenQuery = buf.String() - // return nil - - case sqlparser.TableExprs: - var exprs []string - for i, n := range node { - var s string - // TODO: fix this hack - // prevents indirect expansion - // except on first TableExpr - if i == 0 { - n.Accept(v) - s = v.GetRewrittenQuery() - } else { - nodeVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - n.Accept(nodeVis) - s = nodeVis.GetRewrittenQuery() - } - exprs = append(exprs, s) - } - v.rewrittenQuery = strings.Join(exprs, ", ") - - case *sqlparser.AliasedTableExpr: - var exprStr, partitionStr string - aliasHandledByIndirect := false - if node.Expr != nil { - anCtx, ok := v.annotations[node] - if !ok { - return fmt.Errorf("cannot find annotated tabulation for table object") - } - if anCtx.GetTableMeta().IsOnClauseHoistable() { - v.hoistedOnClauseTables = append(v.hoistedOnClauseTables, node) - } - if indirect, isIndirect := anCtx.GetTableMeta().GetIndirect(); isIndirect { - // - name := indirect.GetName() - indirectType := indirect.GetType() - switch indirectType { - case astindirect.ViewType: - // Use the user-specified alias if present, otherwise the view name. - // The alias is embedded in the template to prevent double aliasing - // when the node.As fallthrough at the end of this case would append it again. - viewAlias := name - if !node.As.IsEmpty() { - viewAlias = node.As.GetRawVal() - } - templateString := fmt.Sprintf(` ( %s ) AS "%s" `, textutil.IndirectQueryPlaceholder, viewAlias) - v.rewrittenQuery = templateString - v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext()) - aliasHandledByIndirect = true - case astindirect.SubqueryType: - // Note: CTEs are converted to SubqueryType at AST level, - // so this path handles both regular subqueries and CTEs. - templateString := " ( " + textutil.IndirectQueryPlaceholder + " ) " - v.rewrittenQuery = templateString - v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext()) - case astindirect.MaterializedViewType, astindirect.PhysicalTableType: - refString := fmt.Sprintf(` %s `, name) - isQuoted, _ := regexp.MatchString(`^".*"$`, name) - isPostgres := v.sqlSystem.GetName() == constants.SQLDialectPostgres - isRelationExported := v.sqlSystem.IsRelationExported(name) - if !isQuoted && !(isPostgres && isRelationExported) { - refString = fmt.Sprintf(` "%s" `, name) - } - alias := "" - if alias != "" { - refString = fmt.Sprintf(` %s AS "%s" `, refString, alias) - } - v.rewrittenQuery = refString - default: - return fmt.Errorf("unknown indirect type '%T'", indirectType) - } - } else { - switch ex := node.Expr.(type) { - case sqlparser.TableName: - if ex.IsEmpty() { - return nil - } - _, isSQLDataSource := anCtx.GetTableMeta().GetSQLDataSource() - var tblStr string - // TODO: clean this garbage up - if isSQLDataSource && !v.isAvoidSQLSourceNaming { - tblStr = anCtx.GetHIDs().GetStackQLTableName() - } else { - dbTbl, err := v.dc.GetCurrentTable(anCtx.GetHIDs()) - if err != nil { - return err - } - tblStr = dbTbl.GetName() - } - fqtn, err := v.sqlSystem.GetFullyQualifiedTableName(tblStr) - v.rewrittenQuery = fqtn - if err != nil { - return err - } - default: - err := node.Expr.Accept(v) - if err != nil { - return err - } - } - } - exprStr = v.GetRewrittenQuery() - } - if node.Partitions != nil { - node.Partitions.Accept(v) - partitionStr = v.GetRewrittenQuery() - } - q := fmt.Sprintf("%s%s", exprStr, partitionStr) - if !node.As.IsEmpty() && !aliasHandledByIndirect { - node.As.Accept(v) - asStr := v.GetRewrittenQuery() - q = fmt.Sprintf("%s as %v", q, asStr) - } - if node.Hints != nil { - node.Hints.Accept(v) - // Hint node provides the space padding. - hintStr := v.GetRewrittenQuery() - q = fmt.Sprintf("%s%v", q, hintStr) - } - v.rewrittenQuery = q - - case sqlparser.TableNames: - var prefix string - for _, n := range node { - n.Accept(v) - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case sqlparser.TableName: - return fmt.Errorf("FromVisitor should not be able to assess an unadorned TableName") - - case *sqlparser.ParenTableExpr: - buf.AstPrintf(node, "(%v)", node.Exprs) - v.rewrittenQuery = buf.String() - - case *sqlparser.NativeQuery: - buf.AstPrintf(node, "NATIVEQUERY '&s'", strings.ReplaceAll(node.QueryString, "'", "''")) - - case sqlparser.JoinCondition: - v.Visit(node.On) - if node.On != nil { - buf.AstPrintf(node, " %v", node.On) - } - if node.Using != nil { - buf.AstPrintf(node, " using %v", node.Using) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.JoinTableExpr: - lVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - lVis.SetAvoidSQLSourceNaming(v.isAvoidSQLSourceNaming) - node.LeftExpr.Accept(lVis) - rVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - rVis.SetAvoidSQLSourceNaming(v.isAvoidSQLSourceNaming) - node.RightExpr.Accept(rVis) - conditionVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - conditionVis.SetAvoidSQLSourceNaming(v.isAvoidSQLSourceNaming) - node.Condition.Accept(conditionVis) - lhsHoistedIntoOn := lVis.GetHoistedOnClauseTables() - rhsHoistedIntoOn := rVis.GetHoistedOnClauseTables() - lhsIndirectContexts := lVis.GetIndirectContexts() - rhsIndirectContexts := rVis.GetIndirectContexts() - v.indirectContexts = append(v.indirectContexts, lhsIndirectContexts...) - v.indirectContexts = append(v.indirectContexts, rhsIndirectContexts...) - // accumulate hoisted tables - v.hoistedOnClauseTables = append(v.hoistedOnClauseTables, lhsHoistedIntoOn...) - v.hoistedOnClauseTables = append(v.hoistedOnClauseTables, rhsHoistedIntoOn...) - joinStr := node.Join - hasCondition := node.Condition.On != nil || node.Condition.Using != nil - var onClause string - switch { - case len(v.hoistedOnClauseTables) > 0 && hasCondition: - onClause = fmt.Sprintf(" ON ( %s ) AND %s", textutil.ControlOnClausePlaceholder, conditionVis.GetRewrittenQuery()) - case len(v.hoistedOnClauseTables) > 0: - onClause = fmt.Sprintf(" ON ( %s )", textutil.ControlOnClausePlaceholder) - case hasCondition: - onClause = fmt.Sprintf(" ON %s", conditionVis.GetRewrittenQuery()) - case joinStr == sqlparser.JoinStr: - // JOIN without ON is a cross join; postgres requires it spelled out - joinStr = crossJoinStr - } - v.rewrittenQuery = fmt.Sprintf("%s %s %s%s", lVis.GetRewrittenQuery(), joinStr, rVis.GetRewrittenQuery(), onClause) - - case *sqlparser.IndexHints: - buf.AstPrintf(node, " %sindex ", node.Type) - if len(node.Indexes) == 0 { - buf.AstPrintf(node, "()") - } else { - prefix := "(" - for _, n := range node.Indexes { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - buf.AstPrintf(node, ")") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.Where: - if node == nil || node.Expr == nil { - return nil - } - buf.AstPrintf(node, " %s %v", node.Type, node.Expr) - v.rewrittenQuery = buf.String() - - case sqlparser.Exprs: - var prefix string - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case *sqlparser.AndExpr: - buf.AstPrintf(node, "%v and %v", node.Left, node.Right) - v.rewrittenQuery = buf.String() - - case *sqlparser.OrExpr: - buf.AstPrintf(node, "%v or %v", node.Left, node.Right) - v.rewrittenQuery = buf.String() - - case *sqlparser.XorExpr: - buf.AstPrintf(node, "%v xor %v", node.Left, node.Right) - v.rewrittenQuery = buf.String() - - case *sqlparser.NotExpr: - buf.AstPrintf(node, "not %v", node.Expr) - v.rewrittenQuery = buf.String() - - case *sqlparser.ComparisonExpr: - lVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - node.Left.Accept(lVis) - rVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.sqlSystem, v.formatter, v.namespaceCollection, v.annotations, v.dc) - node.Right.Accept(rVis) - buf.AstPrintf(node, "%s %s %s", lVis.GetRewrittenQuery(), node.Operator, rVis.GetRewrittenQuery()) - if node.Escape != nil { - buf.AstPrintf(node, " escape %v", node.Escape) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.RangeCond: - buf.AstPrintf(node, "%v %s %v and %v", node.Left, node.Operator, node.From, node.To) - v.rewrittenQuery = buf.String() - - case *sqlparser.IsExpr: - buf.AstPrintf(node, "%v %s", node.Expr, node.Operator) - v.rewrittenQuery = buf.String() - - case *sqlparser.ExistsExpr: - buf.AstPrintf(node, "exists %v", node.Subquery) - v.rewrittenQuery = buf.String() - - case *sqlparser.SQLVal: - switch node.Type { - case sqlparser.StrVal: - sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val).EncodeSQL(buf) - case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.HexNum: - buf.AstPrintf(node, "%s", node.Val) - case sqlparser.HexVal: - buf.AstPrintf(node, "X'%s'", node.Val) - case sqlparser.BitVal: - buf.AstPrintf(node, "B'%s'", node.Val) - case sqlparser.ValArg: - buf.WriteArg(string(node.Val)) - default: - panic("unexpected") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.NullVal: - buf.AstPrintf(node, "null") - v.rewrittenQuery = buf.String() - - case sqlparser.BoolVal: - if node { - buf.AstPrintf(node, "true") - } else { - buf.AstPrintf(node, "false") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.ColName: - if !node.Qualifier.IsEmpty() { - buf.AstPrintf(node, "%v.", node.Qualifier) - } - buf.AstPrintf(node, "%v", node.Name) - v.rewrittenQuery = buf.String() - - case sqlparser.ValTuple: - buf.AstPrintf(node, "(%v)", sqlparser.Exprs(node)) - v.rewrittenQuery = buf.String() - - case *sqlparser.Subquery: - buf.AstPrintf(node, "(%v)", node.Select) - v.rewrittenQuery = buf.String() - - case sqlparser.ListArg: - buf.WriteArg(string(node)) - v.rewrittenQuery = buf.String() - - case *sqlparser.BinaryExpr: - buf.AstPrintf(node, "%v %s %v", node.Left, node.Operator, node.Right) - v.rewrittenQuery = buf.String() - - case *sqlparser.UnaryExpr: - if _, unary := node.Expr.(*sqlparser.UnaryExpr); unary { - // They have same precedence so parenthesis is not required. - buf.AstPrintf(node, "%s %v", node.Operator, node.Expr) - return nil - } - buf.AstPrintf(node, "%s%v", node.Operator, node.Expr) - v.rewrittenQuery = buf.String() - - case *sqlparser.IntervalExpr: - buf.AstPrintf(node, "interval %v %s", node.Expr, node.Unit) - v.rewrittenQuery = buf.String() - - case *sqlparser.TimestampFuncExpr: - buf.AstPrintf(node, "%s(%s, %v, %v)", node.Name, node.Unit, node.Expr1, node.Expr2) - v.rewrittenQuery = buf.String() - - case *sqlparser.CurTimeFuncExpr: - buf.AstPrintf(node, "%s(%v)", node.Name.String(), node.Fsp) - v.rewrittenQuery = buf.String() - - case *sqlparser.CollateExpr: - buf.AstPrintf(node, "%v collate %s", node.Expr, node.Charset) - v.rewrittenQuery = buf.String() - - case *sqlparser.ExecSubquery: - if node.Exec == nil { - return fmt.Errorf("cannont accomodate nil exec table container") - } - // if v.shouldCollectTables { - // switch te := node.Exec.MethodName.(type) { - // case sqlparser.TableName: - // v.tablesCited[node] = te - // } - // } - s := astformat.String(node.Exec.MethodName, v.sqlSystem.GetASTFormatter()) - v.rewrittenQuery = s - - case *sqlparser.FuncExpr: - newNode, err := v.sqlSystem.GetASTFuncRewriter().RewriteFunc(node) - if err != nil { - return err - } - node.Distinct = newNode.Distinct - node.Exprs = newNode.Exprs - node.Name = newNode.Name - var distinct string - if node.Distinct { - distinct = "distinct " - } - if !node.Qualifier.IsEmpty() { - buf.AstPrintf(node, "%v.", node.Qualifier) - } - // Function names should not be back-quoted even - // if they match a reserved word, only if they contain illegal characters - funcName := node.Name.String() - - if sqlparser.ContainEscapableChars(funcName, sqlparser.NoAt) { - sqlparser.WriteEscapedString(buf, funcName) - } else { - buf.WriteString(funcName) - } - buf.AstPrintf(node, "(%s%v)", distinct, node.Exprs) - v.rewrittenQuery = buf.String() - - case *sqlparser.GroupConcatExpr: - buf.AstPrintf(node, "group_concat(%s%v%v%s%v)", node.Distinct, node.Exprs, node.OrderBy, node.Separator, node.Limit) - v.rewrittenQuery = buf.String() - - case *sqlparser.ValuesFuncExpr: - buf.AstPrintf(node, "values(%v)", node.Name) - v.rewrittenQuery = buf.String() - - case *sqlparser.SubstrExpr: - var val interface{} - if node.Name != nil { - val = node.Name - } else { - val = node.StrVal - } - - if node.To == nil { - buf.AstPrintf(node, "substr(%v, %v)", val, node.From) - } else { - buf.AstPrintf(node, "substr(%v, %v, %v)", val, node.From, node.To) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.ConvertExpr: - buf.AstPrintf(node, "convert(%v, %v)", node.Expr, node.Type) - v.rewrittenQuery = buf.String() - - case *sqlparser.ConvertUsingExpr: - buf.AstPrintf(node, "convert(%v using %s)", node.Expr, node.Type) - v.rewrittenQuery = buf.String() - - case *sqlparser.ConvertType: - buf.AstPrintf(node, "%s", node.Type) - if node.Length != nil { - buf.AstPrintf(node, "(%v", node.Length) - if node.Scale != nil { - buf.AstPrintf(node, ", %v", node.Scale) - } - buf.AstPrintf(node, ")") - } - if node.Charset != "" { - buf.AstPrintf(node, "%s %s", node.Operator, node.Charset) - } - v.rewrittenQuery = buf.String() - - case *sqlparser.MatchExpr: - buf.AstPrintf(node, "match(%v) against (%v%s)", node.Columns, node.Expr, node.Option) - v.rewrittenQuery = buf.String() - - case *sqlparser.CaseExpr: - buf.AstPrintf(node, "case sqlparser.") - if node.Expr != nil { - buf.AstPrintf(node, "%v ", node.Expr) - } - for _, when := range node.Whens { - buf.AstPrintf(node, "%v ", when) - } - if node.Else != nil { - buf.AstPrintf(node, "else %v ", node.Else) - } - buf.AstPrintf(node, "end") - v.rewrittenQuery = buf.String() - - case *sqlparser.Default: - buf.AstPrintf(node, "default") - if node.ColName != "" { - buf.WriteString("(") - sqlparser.FormatID(buf, node.ColName, strings.ToLower(node.ColName), sqlparser.NoAt) - buf.WriteString(")") - } - v.rewrittenQuery = buf.String() - - case *sqlparser.When: - buf.AstPrintf(node, "when %v then %v", node.Cond, node.Val) - v.rewrittenQuery = buf.String() - - case sqlparser.GroupBy: - var colz []string - for _, n := range node { - switch n := n.(type) { - case *sqlparser.ColName: - if n.Qualifier.GetRawVal() == "" { - colz = append(colz, v.sqlSystem.DelimitGroupByColumn(n.Name.GetRawVal())) - } else { - colz = append(colz, fmt.Sprintf(`%s.%s`, v.sqlSystem.DelimitGroupByColumn(n.Qualifier.GetRawVal()), v.sqlSystem.DelimitGroupByColumn(n.Name.GetRawVal()))) - } - default: - colz = append(colz, sqlparser.String(n)) - } - } - if len(colz) > 0 { - v.rewrittenQuery = fmt.Sprintf(" group by %s", strings.Join(colz, ", ")) - } else { - v.rewrittenQuery = "" - } - - case sqlparser.OrderBy: - var colz []string - for _, orderNode := range node { - switch n := orderNode.Expr.(type) { - case *sqlparser.ColName: - if n.Qualifier.GetRawVal() == "" { - colz = append(colz, fmt.Sprintf(`%s %s`, v.sqlSystem.DelimitOrderByColumn(n.Name.GetRawVal()), orderNode.Direction)) - } else { - colz = append(colz, fmt.Sprintf(`%s.%s %s`, v.sqlSystem.DelimitOrderByColumn(n.Qualifier.GetRawVal()), v.sqlSystem.DelimitOrderByColumn(n.Name.GetRawVal()), orderNode.Direction)) - } - default: - colz = append(colz, fmt.Sprintf("%s %s", sqlparser.String(n), orderNode.Direction)) - } - } - if len(colz) > 0 { - v.rewrittenQuery = fmt.Sprintf(" order by %s", strings.Join(colz, ", ")) - } else { - v.rewrittenQuery = "" - } - - case *sqlparser.Order: - if node, ok := node.Expr.(*sqlparser.NullVal); ok { - buf.AstPrintf(node, "%v", node) - return nil - } - if node, ok := node.Expr.(*sqlparser.FuncExpr); ok { - if node.Name.Lowered() == "rand" { - buf.AstPrintf(node, "%v", node) - return nil - } - } - - buf.AstPrintf(node, "%v %s", node.Expr, node.Direction) - v.rewrittenQuery = buf.String() - - case *sqlparser.Limit: - if node == nil { - return nil - } - buf.AstPrintf(node, " limit ") - if node.Offset != nil { - buf.AstPrintf(node, "%v, ", node.Offset) - } - buf.AstPrintf(node, "%v", node.Rowcount) - v.rewrittenQuery = buf.String() - - case sqlparser.Values: - prefix := "values " - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case sqlparser.UpdateExprs: - var prefix string - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case *sqlparser.UpdateExpr: - buf.AstPrintf(node, "%v = %v", node.Name, node.Expr) - v.rewrittenQuery = buf.String() - - case sqlparser.SetExprs: - var prefix string - for _, n := range node { - buf.AstPrintf(node, "%s%v", prefix, n) - prefix = ", " - } - v.rewrittenQuery = buf.String() - - case *sqlparser.SetExpr: - if node.Scope != "" { - buf.WriteString(node.Scope) - buf.WriteString(" ") - } - // We don't have to backtick set variable names. - switch { - case node.Name.EqualString("charset") || node.Name.EqualString("names"): - buf.AstPrintf(node, "%s %v", node.Name.String(), node.Expr) - case node.Name.EqualString(sqlparser.TransactionStr): - sqlVal := node.Expr.(*sqlparser.SQLVal) - buf.AstPrintf(node, "%s %s", node.Name.String(), strings.ToLower(string(sqlVal.Val))) - default: - buf.AstPrintf(node, "%v = %v", node.Name, node.Expr) - } - v.rewrittenQuery = buf.String() - - case sqlparser.OnDup: - if node == nil { - return nil - } - buf.AstPrintf(node, " on duplicate key update %v", sqlparser.UpdateExprs(node)) - v.rewrittenQuery = buf.String() - - case sqlparser.ColIdent: - for i := sqlparser.NoAt; i < node.GetAtCount(); i++ { - buf.WriteByte('@') - } - sqlparser.FormatID(buf, node.GetRawVal(), node.Lowered(), node.GetAtCount()) - v.rewrittenQuery = buf.String() - - case sqlparser.TableIdent: - tn := node.GetRawVal() - v.rewrittenQuery = tn - - case *sqlparser.IsolationLevel: - buf.WriteString("isolation level " + node.Level) - v.rewrittenQuery = buf.String() - - case *sqlparser.AccessMode: - buf.WriteString(node.Mode) - v.rewrittenQuery = buf.String() - } - return nil -} diff --git a/internal/stackql/astvisit/internally_routable_typing.go b/internal/stackql/astvisit/internally_routable_typing.go index 1d135656b..85e72ea33 100644 --- a/internal/stackql/astvisit/internally_routable_typing.go +++ b/internal/stackql/astvisit/internally_routable_typing.go @@ -4,19 +4,17 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/internal/stackql/tablenamespace" "github.com/stackql/stackql/internal/stackql/taxonomy" "github.com/stackql/stackql/internal/stackql/typing" - "github.com/stackql/stackql/internal/stackql/util" ) var ( @@ -30,19 +28,15 @@ type InternallyRoutableTypingAstVisitor interface { } type standardInternallyRoutableTypingAstVisitor struct { - handlerCtx handler.HandlerContext - rawQuery string - dc drm.Config - tables taxonomy.TblMap - annotations taxonomy.AnnotationCtxMap - discoGenIDs map[sqlparser.SQLNode]int - annotatedTabulations taxonomy.AnnotatedTabulationMap - columnNames []parserutil.ColumnHandle - columnDescriptors []formulation.ColumnDescriptor - relationalColumns []typing.RelationalColumn - namespaceCollection tablenamespace.Collection - formatter sqlparser.NodeFormatter - annotatedAST annotatedast.AnnotatedAst + handlerCtx handler.HandlerContext + rawQuery string + dc drm.Config + tables taxonomy.TblMap + columnNames []parserutil.ColumnHandle + relationalColumns []typing.RelationalColumn + namespaceCollection tablenamespace.Collection + formatter sqlparser.NodeFormatter + annotatedAST annotatedast.AnnotatedAst // // single threaded, so no mutex protection anonColCounter int @@ -58,13 +52,12 @@ func NewInternallyRoutableTypingAstVisitor( namespaceCollection tablenamespace.Collection, ) InternallyRoutableTypingAstVisitor { rv := &standardInternallyRoutableTypingAstVisitor{ - rawQuery: rawQuery, - annotatedAST: annotatedAST, - handlerCtx: handlerCtx, - tables: tables, - annotatedTabulations: make(taxonomy.AnnotatedTabulationMap), - dc: dc, - namespaceCollection: namespaceCollection, + rawQuery: rawQuery, + annotatedAST: annotatedAST, + handlerCtx: handlerCtx, + tables: tables, + dc: dc, + namespaceCollection: namespaceCollection, } return rv } @@ -102,41 +95,15 @@ func (v *standardInternallyRoutableTypingAstVisitor) getStarColumns( rv = v.dc.ColumnsToRelationalColumns(indirect.GetColumns()) return rv, nil } - schema, _, err := tbl.GetResponseSchemaAndMediaType() - if err != nil { - return nil, err + ids := tbl.GetHeirarchyObjects().GetHeirarchyIDs() + system := v.handlerCtx.GetSQLSystem() + if metadata, exists := system.GetPhysicalTableByName(ids.GetTableName()); exists { + return metadata.GetColumns(), nil } - itemObjS, selectItemsKey, err := tbl.GetSelectSchemaAndObjectPath() - tbl.SetSelectItemsKey(selectItemsKey) - unsuitableSchemaMsg := "standardInternallyRoutableTypingAstVisitor.getStarColumns(): schema unsuitable for select query" - if err != nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) + if metadata, exists := system.GetMaterializedViewByName(ids.GetTableName()); exists { + return metadata.GetColumns(), nil } - if itemObjS == nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) - } - var cols []parserutil.ColumnHandle - colNames := itemObjS.GetAllColumns(util.TrimSelectItemsKey(selectItemsKey)) - for _, v := range colNames { - cols = append(cols, parserutil.NewUnaliasedColumnHandle(v)) - } - var columnDescriptors []formulation.ColumnDescriptor - for _, col := range cols { - columnDescriptors = append( - columnDescriptors, - formulation.NewColumnDescriptor( - col.Alias, - col.Name, - col.Qualifier, - col.DecoratedColumn, - nil, - schema, - col.Val, - ), - ) - } - relationalColumns := v.dc.OpenapiColumnsToRelationalColumns(columnDescriptors) - return relationalColumns, nil + return system.ObtainRelationalColumnsFromExternalSQLtable(ids) } func (v *standardInternallyRoutableTypingAstVisitor) WithFormatter( @@ -370,18 +337,6 @@ func (v *standardInternallyRoutableTypingAstVisitor) Visit(node sqlparser.SQLNod case *sqlparser.AutoIncSpec: - case *sqlparser.ExecSubquery: - t, ok := v.annotations[node] - if !ok { - return fmt.Errorf("exec: could not infer annotated table") - } - dID, ok := v.discoGenIDs[node] - if !ok { - return fmt.Errorf("exec: could not infer discovery generation ID") - } - replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID) - node.Exec.MethodName = replacementExpr - case *sqlparser.VindexSpec: numParams := len(node.Params) @@ -538,9 +493,6 @@ func (v *standardInternallyRoutableTypingAstVisitor) Visit(node sqlparser.SQLNod // broadcastType = v.getTypeFromParserType(expr.Type) // default: // } - // cd := formulation.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, nil, col.Val) - // v.columnDescriptors = append(v.columnDescriptors, cd) - // relCol := v.dc.OpenapiColumnsToRelationalColumn(cd) rv := typing.NewRelationalColumn( col.Name, v.getTypeFromParserType(col.Type), @@ -602,21 +554,23 @@ func (v *standardInternallyRoutableTypingAstVisitor) Visit(node sqlparser.SQLNod logging.GetLogger().Debugf("sqlDataSource = '%v'\n", sqlDataSource) return nil } - schema, err := tbl.GetSelectableObjectSchema() + col, err := parserutil.InferColNameFromExpr(node, v.formatter) if err != nil { return err } - col, err := parserutil.InferColNameFromExpr(node, v.formatter) + v.columnNames = append(v.columnNames, col) + columns, err := v.getStarColumns(tbl) if err != nil { return err } - v.columnNames = append(v.columnNames, col) - ss, _ := schema.GetProperty(col.Name) - cd := formulation.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, ss, col.Val) - v.columnDescriptors = append(v.columnDescriptors, cd) - v.relationalColumns = append(v.relationalColumns, v.dc.OpenapiColumnsToRelationalColumn(cd)) - if !node.As.IsEmpty() { + for _, declared := range columns { + if declared.GetName() == col.Name || declared.GetIdentifier() == col.Name { + v.relationalColumns = append(v.relationalColumns, + declared.WithAlias(col.Alias).WithQualifier(col.Qualifier).WithDecorated(col.DecoratedColumn)) + return nil + } } + return fmt.Errorf("cannot resolve local column %q", col.Name) case sqlparser.Nextval: @@ -642,27 +596,6 @@ func (v *standardInternallyRoutableTypingAstVisitor) Visit(node sqlparser.SQLNod } case *sqlparser.AliasedTableExpr: - if node.Expr != nil && !parserutil.IsNullTable(node) { - switch node.Expr.(type) { - case sqlparser.TableName: - t, ok := v.annotations[node] - if !ok { - return fmt.Errorf("could not infer annotated table") - } - _, isIndirect := t.GetTableMeta().GetIndirect() - _, isSQLDataSource := t.GetTableMeta().GetSQLDataSource() - if isIndirect || isSQLDataSource { - // do nothing - } else { - dID, ok := v.discoGenIDs[node] - if !ok { - return fmt.Errorf("could not infer discovery generation ID") - } - replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID) - node.Expr = replacementExpr - } - } - } if node.Partitions != nil { node.Partitions.Accept(v) } diff --git a/internal/stackql/astvisit/param_extract.go b/internal/stackql/astvisit/param_extract.go index 865d4e2d7..817bf6ec4 100644 --- a/internal/stackql/astvisit/param_extract.go +++ b/internal/stackql/astvisit/param_extract.go @@ -4,10 +4,10 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/sqltypes" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" ) diff --git a/internal/stackql/astvisit/query_rewriting.go b/internal/stackql/astvisit/query_rewriting.go deleted file mode 100644 index 8998d2032..000000000 --- a/internal/stackql/astvisit/query_rewriting.go +++ /dev/null @@ -1,1129 +0,0 @@ -package astvisit - -import ( - "fmt" - "regexp" - "sort" - "strings" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/sqlrewrite" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/internal/stackql/typing" - "github.com/stackql/stackql/internal/stackql/util" -) - -var ( - _ QueryRewriteAstVisitor = &standardQueryRewriteAstVisitor{} - isJSONEachCompatibleRegexp *regexp.Regexp = regexp.MustCompile( - `^(.*\.value|value)$`) -) - -type QueryRewriteAstVisitor interface { - sqlparser.SQLAstVisitor - GenerateSelectDML() (drm.PreparedStatementCtx, error) - WithFormatter(formatter sqlparser.NodeFormatter) QueryRewriteAstVisitor - WithPrepStmtOffset(int) QueryRewriteAstVisitor -} - -type standardQueryRewriteAstVisitor struct { - handlerCtx handler.HandlerContext - dc drm.Config - tables taxonomy.TblMap - annotations taxonomy.AnnotationCtxMap - discoGenIDs map[sqlparser.SQLNode]int - annotatedTabulations taxonomy.AnnotatedTabulationMap - selectCtx drm.PreparedStatementCtx - baseCtrlCounters internaldto.TxnControlCounters - secondaryCtrlCounters []internaldto.TxnControlCounters - colRefs parserutil.ColTableMap - columnNames []parserutil.ColumnHandle - columnDescriptors []formulation.ColumnDescriptor - relationalColumns []typing.RelationalColumn - tableSlice []tableinsertioncontainer.TableInsertionContainer - namespaceCollection tablenamespace.Collection - formatter sqlparser.NodeFormatter - annotatedAST annotatedast.AnnotatedAst - // - fromStr string - whereExprsStr string - selectSuffix string - // singe threaded, so no mutex protection - anonColCounter int - // - indirectContexts []drm.PreparedStatementCtx - // - prepStmtOffset int - hoistedOnClauseTables []sqlparser.SQLNode -} - -func NewQueryRewriteAstVisitor( - annotatedAST annotatedast.AnnotatedAst, - handlerCtx handler.HandlerContext, - tables taxonomy.TblMap, - tableSlice []tableinsertioncontainer.TableInsertionContainer, - annotations taxonomy.AnnotationCtxMap, - discoGenIDs map[sqlparser.SQLNode]int, - colRefs parserutil.ColTableMap, - dc drm.Config, - txnCtrlCtrs internaldto.TxnControlCounters, - secondaryTccs []internaldto.TxnControlCounters, - rewrittenWhere string, - namespaceCollection tablenamespace.Collection, -) QueryRewriteAstVisitor { - rv := &standardQueryRewriteAstVisitor{ - annotatedAST: annotatedAST, - handlerCtx: handlerCtx, - tables: tables, - tableSlice: tableSlice, - annotations: annotations, - discoGenIDs: discoGenIDs, - annotatedTabulations: make(taxonomy.AnnotatedTabulationMap), - colRefs: colRefs, - dc: dc, - baseCtrlCounters: txnCtrlCtrs, - secondaryCtrlCounters: secondaryTccs, - whereExprsStr: rewrittenWhere, - namespaceCollection: namespaceCollection, - } - return rv -} - -func (v *standardQueryRewriteAstVisitor) isJSONEachCompatible(col parserutil.ColumnHandle) bool { - s := col.Name - qualifier := col.Qualifier - if qualifier != "" { - s = fmt.Sprintf("%s.%s", qualifier, s) - } - return isJSONEachCompatibleRegexp.MatchString(s) -} - -func (v *standardQueryRewriteAstVisitor) isExpressionProjection(node *sqlparser.AliasedExpr) bool { - _, isColumnReference := node.Expr.(*sqlparser.ColName) - return !isColumnReference -} - -// cloneRelationalColumn returns a fresh column carrying the catalogued identity -// (name, type, width, oid) of col, so that projection-level decoration such as -// an alias or qualifier never mutates the relation's shared catalogue entry. -func cloneRelationalColumn(col typing.RelationalColumn) typing.RelationalColumn { - rv := typing.NewRelationalColumn(col.GetName(), col.GetType()).WithWidth(col.GetWidth()) - if oID, hasOID := col.GetOID(); hasOID { - rv = rv.WithOID(oID) - } - return rv -} - -// TODO: introduce dependency on RDBMS -func (v *standardQueryRewriteAstVisitor) getTypeFromParserType(t sqlparser.ValType) string { - //nolint:exhaustive // acceptable - switch t { - case sqlparser.StrVal: - return "string" - case sqlparser.IntVal: - return "int" - case sqlparser.FloatVal: - return "float" - default: - return "string" - } -} - -func (v *standardQueryRewriteAstVisitor) WithPrepStmtOffset(offset int) QueryRewriteAstVisitor { - v.prepStmtOffset = offset - return v -} - -func (v *standardQueryRewriteAstVisitor) getNextAlias() string { - v.anonColCounter++ - i := v.anonColCounter - return fmt.Sprintf("col_%d", i) -} - -func (v *standardQueryRewriteAstVisitor) generateServerVarColumnDescriptor( - k string, m formulation.OperationStore) formulation.ColumnDescriptor { - schema := formulation.NewStringSchema( - m.GetService(), - "", - "", - ) - colDesc := formulation.NewColumnDescriptor( - "", - k, - "", - "", - nil, - schema, - nil, - ) - return colDesc -} - -//nolint:gocognit // acceptable -func (v *standardQueryRewriteAstVisitor) getStarColumns( - tbl tablemetadata.ExtendedTableMetadata, -) ([]typing.RelationalColumn, error) { - existingColumns := make(map[string]struct{}) - if indirect, isIndirect := tbl.GetIndirect(); isIndirect { - rv := indirect.GetRelationalColumns() - if len(rv) > 0 { - return rv, nil - } - rv = v.dc.ColumnsToRelationalColumns(indirect.GetColumns()) - return rv, nil - } - schema, _, err := tbl.GetResponseSchemaAndMediaType() - if err != nil { - return nil, err - } - itemObjS, selectItemsKey, err := tbl.GetSelectSchemaAndObjectPath() - tbl.SetSelectItemsKey(selectItemsKey) - unsuitableSchemaMsg := "standardQueryRewriteAstVisitor.getStarColumns(): schema unsuitable for select query" - if err != nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) - } - if itemObjS == nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) - } - var cols []parserutil.ColumnHandle - colNames := itemObjS.GetAllColumns(util.TrimSelectItemsKey(selectItemsKey)) - for _, v := range colNames { - existingColumns[v] = struct{}{} - cols = append(cols, parserutil.NewUnaliasedColumnHandle(v)) - } - var columnDescriptors []formulation.ColumnDescriptor - for _, col := range cols { - columnDescriptors = append( - columnDescriptors, - formulation.NewColumnDescriptor( - col.Alias, - col.Name, - col.Qualifier, - col.DecoratedColumn, - nil, - schema, - col.Val, - ), - ) - } - m, mErr := tbl.GetMethod() - if m != nil && mErr == nil { - servers, serversDoExist := m.GetServers() - if serversDoExist { - for _, srv := range servers { - for k := range srv.Variables { - if _, ok := existingColumns[k]; ok { - continue - } - existingColumns[k] = struct{}{} - colDesc := v.generateServerVarColumnDescriptor(k, m) - columnDescriptors = append(columnDescriptors, colDesc) - } - } - } - } - relationalColumns := v.dc.OpenapiColumnsToRelationalColumns(columnDescriptors) - sort.Sort(typing.RelationalColumnByName(relationalColumns)) - return relationalColumns, nil -} - -func (v *standardQueryRewriteAstVisitor) GenerateSelectDML() (drm.PreparedStatementCtx, error) { - relationalColumns := v.relationalColumns - rewriteInput := sqlrewrite.NewStandardSQLRewriteInput( - v.dc, - relationalColumns, - v.baseCtrlCounters, - v.selectSuffix, - v.whereExprsStr, - v.secondaryCtrlCounters, - v.tables, - v.fromStr, - v.tableSlice, - v.namespaceCollection, - v.hoistedOnClauseTables, - make(map[string]interface{}), - ).WithIndirectContexts(v.indirectContexts).WithPrepStmtOffset( - v.prepStmtOffset + v.annotatedAST.GetSubequeryTableCount()) - return sqlrewrite.GenerateRewrittenSelectDML(rewriteInput) -} - -// Need not be view-aware. - -func (v *standardQueryRewriteAstVisitor) WithFormatter(formatter sqlparser.NodeFormatter) QueryRewriteAstVisitor { - v.formatter = formatter - return v -} - -func (v *standardQueryRewriteAstVisitor) GetTableMap() taxonomy.TblMap { - return v.tables -} - -func (v *standardQueryRewriteAstVisitor) GetColumnDescriptors() []formulation.ColumnDescriptor { - return v.columnDescriptors -} - -func (v *standardQueryRewriteAstVisitor) GetSelectContext() (drm.PreparedStatementCtx, bool) { - if v.selectCtx != nil { - return v.selectCtx, true - } - return nil, false -} - -//nolint:dupl,funlen,gocognit,gocyclo,cyclop,errcheck,staticcheck,gocritic,lll,govet,nestif,exhaustive,gomnd,revive // defer uplifts on analysers -func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error { - var err error - - switch node := node.(type) { - case *sqlparser.Select: - v.selectSuffix = GenerateModifiedSelectSuffix(v.annotatedAST, node, v.handlerCtx.GetSQLSystem(), v.handlerCtx.GetASTFormatter(), v.namespaceCollection) - var options string - addIf := func(b bool, s string) { - if b { - options += s - } - } - addIf(node.Distinct, sqlparser.DistinctStr) - if node.Cache != nil { - if *node.Cache { - options += sqlparser.SQLCacheStr - } else { - options += sqlparser.SQLNoCacheStr - } - } - addIf(node.StraightJoinHint, sqlparser.StraightJoinHint) - addIf(node.SQLCalcFoundRows, sqlparser.SQLCalcFoundRowsStr) - - if node.Comments != nil { - node.Comments.Accept(v) - } - if node.SelectExprs != nil { - err = node.SelectExprs.Accept(v) - if err != nil { - return err - } - } - if node.From != nil { - err := node.From.Accept(v) - if err != nil { - return err - } - fromVis := NewFromRewriteAstVisitor(v.annotatedAST, "", true, v.handlerCtx.GetSQLSystem(), v.formatter, v.namespaceCollection, v.annotations, v.dc) - fromVis.SetAvoidSQLSourceNaming(true) - if node.From != nil { - node.From.Accept(fromVis) - v.fromStr = fromVis.GetRewrittenQuery() - v.hoistedOnClauseTables = append(v.hoistedOnClauseTables, fromVis.GetHoistedOnClauseTables()...) - } - v.indirectContexts = fromVis.GetIndirectContexts() - } - if node.Where != nil { - node.Where.Accept(v) - } - if node.GroupBy != nil { - node.GroupBy.Accept(v) - } - if node.Having != nil { - node.Having.Accept(v) - } - if node.OrderBy != nil { - node.OrderBy.Accept(v) - } - if node.Limit != nil { - node.Limit.Accept(v) - } - return nil - - case *sqlparser.ParenSelect: - node.Accept(v) - - case *sqlparser.Auth: - return nil - - case *sqlparser.AuthRevoke: - return nil - case *sqlparser.Sleep: - return nil - - case *sqlparser.Union: - err = node.FirstStatement.Accept(v) - if err != nil { - return err - } - for _, us := range node.UnionSelects { - err = us.Accept(v) - if err != nil { - return err - } - } - return nil - - case *sqlparser.UnionSelect: - return node.Statement.Accept(v) - - case *sqlparser.Stream: - err = node.Comments.Accept(v) - if err != nil { - return err - } - err = node.SelectExpr.Accept(v) - if err != nil { - return err - } - return node.Table.Accept(v) - - case *sqlparser.Insert: - - case *sqlparser.Update: - - case *sqlparser.Delete: - - case *sqlparser.Set: - - case *sqlparser.SetTransaction: - - case *sqlparser.DBDDL: - - case *sqlparser.DDL: - switch node.Action { - case sqlparser.CreateStr: - case sqlparser.DropStr: - case sqlparser.RenameStr: - case sqlparser.AlterStr: - case sqlparser.FlushStr: - case sqlparser.CreateVindexStr: - case sqlparser.DropVindexStr: - case sqlparser.AddVschemaTableStr: - case sqlparser.DropVschemaTableStr: - case sqlparser.AddColVindexStr: - case sqlparser.DropColVindexStr: - case sqlparser.AddSequenceStr: - case sqlparser.AddAutoIncStr: - default: - } - - case *sqlparser.OptLike: - - case *sqlparser.PartitionSpec: - switch node.Action { - case sqlparser.ReorganizeStr: - default: - } - - case *sqlparser.PartitionDefinition: - - case *sqlparser.TableSpec: - - case *sqlparser.ColumnDefinition: - - // Format returns a canonical string representation of the type and all relevant options - case *sqlparser.ColumnType: - ct := node - - if ct.Length != nil && ct.Scale != nil { - - } else if ct.Length != nil { - } - - if ct.EnumValues != nil { - } - - opts := make([]string, 0, 16) //nolint:mnd // TODO: tech debt sweep mnd hacks - if ct.Unsigned { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNSIGNED]) - } - if ct.Zerofill { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ZEROFILL]) - } - if ct.Charset != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.CHARACTER], sqlparser.KeywordStrings[sqlparser.SET], ct.Charset) - } - if ct.Collate != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COLLATE], ct.Collate) - } - if ct.NotNull { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.NOT], sqlparser.KeywordStrings[sqlparser.NULL]) - } - if ct.Default != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.DEFAULT], sqlparser.String(ct.Default)) - } - if ct.OnUpdate != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ON], sqlparser.KeywordStrings[sqlparser.UPDATE], sqlparser.String(ct.OnUpdate)) - } - if ct.Autoincrement { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.AUTO_INCREMENT]) - } - if ct.Comment != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COMMENT_KEYWORD], sqlparser.String(ct.Comment)) - } - if ct.KeyOpt == sqlparser.ColKeyPrimary { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.PRIMARY], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeyUnique { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE]) - } - if ct.KeyOpt == sqlparser.ColKeyUniqueKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeySpatialKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.SPATIAL], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.KEY]) - } - - if len(opts) != 0 { - } - - case *sqlparser.IndexDefinition: - idx := node - for i, col := range idx.Columns { - if i != 0 { - } else { - } - if col.Length != nil { - } - } - - for _, opt := range idx.Options { - if opt.Using != "" { - } else { - } - } - - case *sqlparser.IndexInfo: - ii := node - if ii.Primary { - } else { - if !ii.Name.IsEmpty() { - } - } - - case *sqlparser.AutoIncSpec: - - case *sqlparser.ExecSubquery: - t, ok := v.annotations[node] - if !ok { - return fmt.Errorf("exec: could not infer annotated table") - } - dID, ok := v.discoGenIDs[node] - if !ok { - return fmt.Errorf("exec: could not infer discovery generation ID") - } - replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID) - node.Exec.MethodName = replacementExpr - - case *sqlparser.VindexSpec: - - numParams := len(node.Params) - if numParams != 0 { - for i, p := range node.Params { - logging.GetLogger().Debugf("%v\n", p) - if i != 0 { - } - } - } - - case sqlparser.VindexParam: - - case *sqlparser.ConstraintDefinition: - c := node - if c.Name != "" { - } - - case sqlparser.ReferenceAction: - a := node - switch a { - case sqlparser.Restrict: - case sqlparser.Cascade: - case sqlparser.NoAction: - case sqlparser.SetNull: - case sqlparser.SetDefault: - } - - case *sqlparser.ForeignKeyDefinition: - f := node - if f.OnDelete != sqlparser.DefaultAction { - } - if f.OnUpdate != sqlparser.DefaultAction { - } - - case *sqlparser.Show: - nodeType := strings.ToLower(node.Type) - if (nodeType == "tables" || nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.ShowTablesOpt != nil { - opt := node.ShowTablesOpt - if node.Extended != "" { - } else { - } - if (nodeType == "columns" || nodeType == "fields") && node.HasOnTable() { - } - if (nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.HasOnTable() { - } - if opt.DbName != "" { - } - return nil - } - if node.Scope == "" { - } else { - } - if node.HasOnTable() { - } - if nodeType == "collation" && node.ShowCollationFilterOpt != nil { - } - if nodeType == "charset" && node.ShowTablesOpt != nil { - } - if node.HasTable() { - } - - case *sqlparser.ShowFilter: - if node == nil { - return nil - } - if node.Like != "" { - } else { - } - - case *sqlparser.Use: - if node.DBName.GetRawVal() != "" { - } else { - } - - case *sqlparser.Commit: - - case *sqlparser.Begin: - - case *sqlparser.Rollback: - - case *sqlparser.SRollback: - - case *sqlparser.Savepoint: - - case *sqlparser.Release: - - case *sqlparser.Explain: - switch node.Type { - case "": // do nothing - case sqlparser.AnalyzeStr: - default: - } - - case *sqlparser.OtherRead: - - case *sqlparser.DescribeTable: - - case *sqlparser.DescribeMethod: - - case *sqlparser.OtherAdmin: - - case sqlparser.Comments: - - case sqlparser.SelectExprs: - for _, n := range node { - err = v.Visit(n) - if err != nil { - return err - } - } - - case *sqlparser.StarExpr: - var tbl tablemetadata.ExtendedTableMetadata - if node.TableName.IsEmpty() { - if len(v.tables) != 1 { - return fmt.Errorf("unaliased star expr not permitted for table count = %d", len(v.tables)) - } - for _, v := range v.tables { - tbl = v - break - } - } else { - var ok bool - tbl, ok = v.tables[node.TableName] - if !ok { - return fmt.Errorf("could not locate table for expr '%v'", node.TableName) - } - } - cols, err := v.getStarColumns(tbl) - if err != nil { - return err - } - v.relationalColumns = append(v.relationalColumns, cols...) - - case *sqlparser.AliasedExpr: - tbl, tblErr := v.tables.GetTableLoose(node) - err := v.Visit(node.Expr) - if err != nil { - return err - } - if tblErr != nil { - col, err := parserutil.InferColNameFromExpr(node, v.formatter) - if err != nil { - return err - } - if col.Alias == "" && (col.Name == "" || strings.Contains(col.Name, " ")) { - col.Alias = v.getNextAlias() - } - v.columnNames = append(v.columnNames, col) - cd := formulation.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, nil, col.Val) - v.columnDescriptors = append(v.columnDescriptors, cd) - v.relationalColumns = append(v.relationalColumns, v.dc.OpenapiColumnsToRelationalColumn(cd)) - return nil - } - if indirect, isIndirect := tbl.GetIndirect(); isIndirect { - col, err := parserutil.InferColNameFromExpr(node, v.formatter) - if err != nil { - return err - } - if col.IsAggregateExpr { - rv := typing.NewRelationalColumn( - col.Name, - v.getTypeFromParserType(col.Type), - ).WithDecorated( - col.DecoratedColumn, - ).WithAlias( - col.Alias, - ).WithUnquote(true) - v.relationalColumns = append(v.relationalColumns, rv) - return nil - } - - // Only a bare column reference binds to the relation's catalogued - // column. An expression over a column (CASE, function call, - // concatenation, ...) is not that column: binding it collapsed the - // projection to the referenced column (issue #798). The catalogued - // column is shared across the whole statement, so it is copied - // before per-projection decoration is applied. - var relationalCol typing.RelationalColumn - var ok bool - if !v.isExpressionProjection(node) { - var catalogueCol typing.RelationalColumn - catalogueCol, ok = indirect.GetRelationalColumnByIdentifier(col.Name) - if ok { - relationalCol = cloneRelationalColumn(catalogueCol) - } - } - if !ok { - if col.Val != nil { - relationalCol = typing.NewRelationalColumn(col.Name, v.getTypeFromParserType(col.Type)).WithDecorated(col.DecoratedColumn) - } else { - _, isConvert := col.Expr.(*sqlparser.ConvertExpr) - if isConvert { - convertCol, convertColErr := parserutil.InferColNameFromExpr(node, v.formatter) - if convertColErr != nil { - return convertColErr - } - relationalCol = typing.NewRelationalColumn(convertCol.Name, v.getTypeFromParserType(col.Type)).WithDecorated(convertCol.DecoratedColumn) - } else { - r, ok := indirect.GetColumnByName(col.Name) - if !ok { - if !v.isJSONEachCompatible(col) && !v.isExpressionProjection(node) { - return fmt.Errorf("query rewriting for indirection: cannot find col = '%s'", col.Name) - } - relationalCol = typing.NewRelationalColumn(col.Name, "").WithDecorated(col.DecoratedColumn) // TOOO: clean this up - } else { - relationalCol = typing.NewRelationalColumn(col.Name, r.GetType()).WithDecorated(col.DecoratedColumn) - } - } - } - } - relationalCol = relationalCol.WithAlias(col.Alias) - if col.Qualifier != "" { - relationalCol = relationalCol.WithQualifier(col.Qualifier) - } - v.relationalColumns = append(v.relationalColumns, relationalCol) - return nil - } - // TODO: accomodate SQL data source - sqlDataSource, isSQLDataSource := tbl.GetSQLDataSource() - if isSQLDataSource { - // - col, err := parserutil.InferColNameFromExpr(node, v.formatter) - if err != nil { - return err - } - relationalColumn, err := v.dc.GetSQLSystem().ObtainRelationalColumnFromExternalSQLtable( - tbl.GetHeirarchyObjects().GetHeirarchyIDs(), - col.Name, - ) - if err != nil { - return err - } - relationalColumn = relationalColumn.WithAlias(col.Alias) - relationalColumn = relationalColumn.WithQualifier(col.Qualifier) - v.relationalColumns = append(v.relationalColumns, relationalColumn) - logging.GetLogger().Debugf("sqlDataSource = '%v'\n", sqlDataSource) - return nil - } - schema, err := tbl.GetSelectableObjectSchema() - if err != nil { - return err - } - col, err := parserutil.InferColNameFromExpr(node, v.formatter) - if err != nil { - return err - } - v.columnNames = append(v.columnNames, col) - // The provider schema types the projection ONLY when the expression is - // the bare column itself: a function's RESULT is not its argument's - // column, and binding the argument's schema made type-changing - // functions (typeof, date, ...) yield 0/null (issue #687). Non-bare - // expressions default to the backend's text type, which scans - // dynamically. - var ss formulation.Schema - _, isBareCol := node.Expr.(*sqlparser.ColName) - if isBareCol { - ss, _ = schema.GetProperty(col.Name) - if ss != nil { - // A wire-spelled projection selects the display column (any-sdk #131). - col = col.WithDisplayName(schema.Tabulate(false, "")) - } - } - cd := formulation.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, ss, col.Val) - v.columnDescriptors = append(v.columnDescriptors, cd) - relationalColumn := v.dc.OpenapiColumnsToRelationalColumn(cd) - if !isBareCol && relationalColumn.GetType() == "" { - relationalColumn = typing.NewRelationalColumn( - col.Name, - v.dc.GetRelationalType("string"), - ).WithQualifier(col.Qualifier).WithAlias(col.Alias).WithDecorated(col.DecoratedColumn).WithParserNode(node) - } - v.relationalColumns = append(v.relationalColumns, relationalColumn) - if !node.As.IsEmpty() { - } - - case sqlparser.Nextval: - - case sqlparser.Columns: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.Partitions: - if node == nil { - return nil - } - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.TableExprs: - for _, n := range node { - err := n.Accept(v) - if err != nil { - return err - } - } - - case *sqlparser.AliasedTableExpr: - if node.Expr != nil && !parserutil.IsNullTable(node) { - switch node.Expr.(type) { - case sqlparser.TableName: - t, ok := v.annotations[node] - if !ok { - return fmt.Errorf("could not infer annotated table") - } - _, isIndirect := t.GetTableMeta().GetIndirect() - _, isSQLDataSource := t.GetTableMeta().GetSQLDataSource() - if isIndirect || isSQLDataSource { - // do nothing - } else { - dID, ok := v.discoGenIDs[node] - if !ok { - return fmt.Errorf("could not infer discovery generation ID") - } - replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID) - node.Expr = replacementExpr - } - } - } - if node.Partitions != nil { - node.Partitions.Accept(v) - } - if !node.As.IsEmpty() { - node.As.Accept(v) - } - if node.Hints != nil { - node.Hints.Accept(v) - } - - case sqlparser.TableNames: - for _, n := range node { - n.Accept(v) - } - - case sqlparser.TableName: - if node.IsEmpty() { - return nil - } - if !node.QualifierThird.IsEmpty() { - } - if !node.QualifierSecond.IsEmpty() { - } - if !node.Qualifier.IsEmpty() { - } - - case *sqlparser.ParenTableExpr: - - case sqlparser.JoinCondition: - if node.On != nil { - } - if node.Using != nil { - } - - case *sqlparser.JoinTableExpr: - err = node.LeftExpr.Accept(v) - if err != nil { - return err - } - err = node.RightExpr.Accept(v) - if err != nil { - return err - } - - case *sqlparser.IndexHints: - if len(node.Indexes) == 0 { - } else { - for _, n := range node.Indexes { - logging.GetLogger().Debugf("%v\n", n) - } - } - - case *sqlparser.Where: - if node == nil || node.Expr == nil { - return nil - } - return node.Expr.Accept(v) - - case sqlparser.Exprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.AndExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.OrExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.XorExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.NotExpr: - return node.Expr.Accept(v) - - case *sqlparser.ComparisonExpr: - switch left := node.Left.(type) { - case *sqlparser.ColName: - if b, ok := left.Metadata.(bool); ok && b { - buf := sqlparser.NewTrackedBuffer(nil) - node.Format(buf) - } - } - - case *sqlparser.RangeCond: - - case *sqlparser.IsExpr: - return node.Expr.Accept(v) - - case *sqlparser.ExistsExpr: - return nil - - case *sqlparser.SQLVal: - switch node.Type { - case sqlparser.StrVal: - case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.HexNum: - case sqlparser.HexVal: - case sqlparser.BitVal: - case sqlparser.ValArg: - default: - } - - case *sqlparser.NullVal: - - case sqlparser.BoolVal: - if node { - } else { - } - - case *sqlparser.ColName: - if !node.Qualifier.IsEmpty() { - } - - case sqlparser.ValTuple: - - case *sqlparser.Subquery: - - case sqlparser.ListArg: - - case *sqlparser.BinaryExpr: - - case *sqlparser.UnaryExpr: - if _, unary := node.Expr.(*sqlparser.UnaryExpr); unary { - // They have same precedence so parenthesis is not required. - return nil - } - - case *sqlparser.IntervalExpr: - - case *sqlparser.TimestampFuncExpr: - - case *sqlparser.CurTimeFuncExpr: - - case *sqlparser.CollateExpr: - - case *sqlparser.FuncExpr: - newNode, err := v.dc.GetSQLSystem().GetASTFuncRewriter().RewriteFunc(node) - if err != nil { - return err - } - node.Distinct = newNode.Distinct - node.Exprs = newNode.Exprs - node.Name = newNode.Name - if node.Distinct { - } - if !node.Qualifier.IsEmpty() { - } - // Function names should not be back-quoted even - // if they match a reserved word, only if they contain illegal characters - funcName := node.Name.String() - - if sqlparser.ContainEscapableChars(funcName, sqlparser.NoAt) { - } else { - } - - case *sqlparser.GroupConcatExpr: - // v.handlerCtx.GetSQLSystem().GetASTFuncRewriter().RewriteFunc(node) - - case *sqlparser.ValuesFuncExpr: - - case *sqlparser.SubstrExpr: - if node.Name != nil { - } else { - } - - if node.To == nil { - } else { - } - - case *sqlparser.ConvertExpr: - - case *sqlparser.ConvertUsingExpr: - - case *sqlparser.ConvertType: - if node.Length != nil { - if node.Scale != nil { - } - } - if node.Charset != "" { - } - - case *sqlparser.MatchExpr: - - case *sqlparser.CaseExpr: - if node.Expr != nil { - } - for _, when := range node.Whens { - logging.GetLogger().Debugf("%v\n", when) - } - if node.Else != nil { - } - - case *sqlparser.Default: - if node.ColName != "" { - } - - case *sqlparser.When: - - case sqlparser.GroupBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.OrderBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.Order: - if node, ok := node.Expr.(*sqlparser.NullVal); ok { - logging.GetLogger().Debugf("%v\n", node) - return nil - } - if node, ok := node.Expr.(*sqlparser.FuncExpr); ok { - if node.Name.Lowered() == "rand" { - return nil - } - } - - case *sqlparser.Limit: - if node == nil { - return nil - } - if node.Offset != nil { - } - - case sqlparser.Values: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.UpdateExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.UpdateExpr: - - case sqlparser.SetExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.SetExpr: - if node.Scope != "" { - } - // We don't have to backtick set variable names. - switch { - case node.Name.EqualString("charset") || node.Name.EqualString("names"): - case node.Name.EqualString(sqlparser.TransactionStr): - default: - } - - case sqlparser.OnDup: - if node == nil { - return nil - } - - case sqlparser.ColIdent: - for i := sqlparser.NoAt; i < node.GetAtCount(); i++ { - } - - case sqlparser.TableIdent: - - case *sqlparser.IsolationLevel: - - case *sqlparser.AccessMode: - } - return nil -} diff --git a/internal/stackql/astvisit/table_alias_pairing.go b/internal/stackql/astvisit/table_alias_pairing.go index 9424aedb0..0462f49e3 100644 --- a/internal/stackql/astvisit/table_alias_pairing.go +++ b/internal/stackql/astvisit/table_alias_pairing.go @@ -6,8 +6,8 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" ) diff --git a/internal/stackql/astvisit/table_extract.go b/internal/stackql/astvisit/table_extract.go index 6a5cfdeb0..0e266ff91 100644 --- a/internal/stackql/astvisit/table_extract.go +++ b/internal/stackql/astvisit/table_extract.go @@ -6,8 +6,8 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" ) diff --git a/internal/stackql/asynccompose/async_compose.go b/internal/stackql/asynccompose/async_compose.go deleted file mode 100644 index 1921083c3..000000000 --- a/internal/stackql/asynccompose/async_compose.go +++ /dev/null @@ -1,44 +0,0 @@ -package asynccompose - -import ( - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/provider" -) - -func ComposeAsyncMonitor( - handlerCtx handler.HandlerContext, - precursor primitive.IPrimitive, - prov provider.IProvider, - method formulation.OperationStore, - commentDirectives sqlparser.CommentDirectives, - isReturning bool, - insertCtx drm.PreparedStatementCtx, - drmCfg drm.Config, -) (primitive.IPrimitive, error) { - asm, err := NewAsyncMonitor(handlerCtx, prov, method, isReturning) - if err != nil { - return nil, err - } - // might be pointless - _, err = handlerCtx.GetAuthContext(prov.GetProviderString()) - if err != nil { - return nil, err - } - // - pl := internaldto.NewBasicPrimitiveContext( - handlerCtx.GetAuthContext, - handlerCtx.GetOutfile(), - handlerCtx.GetOutErrFile(), - ) - primitive, err := asm.GetMonitorPrimitive( - prov, method, precursor, pl, commentDirectives, isReturning, insertCtx, drmCfg) - if err != nil { - return nil, err - } - return primitive, err -} diff --git a/internal/stackql/asynccompose/asyncmonitor.go b/internal/stackql/asynccompose/asyncmonitor.go deleted file mode 100644 index 9ffdbb827..000000000 --- a/internal/stackql/asynccompose/asyncmonitor.go +++ /dev/null @@ -1,209 +0,0 @@ -package asynccompose - -import ( - "fmt" - "strings" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/provider" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -type IAsyncMonitor interface { - GetMonitorPrimitive( - prov provider.IProvider, - op formulation.OperationStore, - precursor primitive.IPrimitive, - initialCtx primitive.IPrimitiveCtx, - comments sqlparser.CommentDirectives, - isReturning bool, - insertCtx drm.PreparedStatementCtx, - drmCfg drm.Config, - ) (primitive.IPrimitive, error) -} - -//nolint:unused // TODO: refactor -type AsyncHTTPMonitorPrimitive struct { - handlerCtx handler.HandlerContext - prov provider.IProvider - op formulation.OperationStore - initialCtx primitive.IPrimitiveCtx - precursor primitive.IPrimitive - executor func(pc primitive.IPrimitiveCtx, initalBody interface{}) internaldto.ExecutorOutput - elapsedSeconds int - pollIntervalSeconds int - noStatus bool - id int64 - comments sqlparser.CommentDirectives -} - -func (pr *AsyncHTTPMonitorPrimitive) SetTxnID(_ int) { -} - -func (pr *AsyncHTTPMonitorPrimitive) IsReadOnly() bool { - return false -} - -func (pr *AsyncHTTPMonitorPrimitive) GetRedoLog() (binlog.LogEntry, bool) { - return nil, false -} - -func (pr *AsyncHTTPMonitorPrimitive) GetUndoLog() (binlog.LogEntry, bool) { - return nil, false -} - -func (pr *AsyncHTTPMonitorPrimitive) WithDebugName(_ string) primitive.IPrimitive { - return pr -} - -func (pr *AsyncHTTPMonitorPrimitive) SetUndoLog(_ binlog.LogEntry) { -} - -func (pr *AsyncHTTPMonitorPrimitive) SetRedoLog(_ binlog.LogEntry) { -} - -func (pr *AsyncHTTPMonitorPrimitive) IncidentData(fromID int64, input internaldto.ExecutorOutput) error { - return pr.precursor.IncidentData(fromID, input) -} - -func (pr *AsyncHTTPMonitorPrimitive) SetInputAlias(alias string, id int64) error { - return pr.precursor.SetInputAlias(alias, id) -} - -func (pr *AsyncHTTPMonitorPrimitive) Optimise() error { - return nil -} - -func (pr *AsyncHTTPMonitorPrimitive) Execute(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - if pr.executor != nil { - if pc == nil { - pc = pr.initialCtx - } - subPr := pr.precursor.Execute(pc) - if subPr.GetError() != nil || pr.executor == nil { - return subPr - } - prStr := pr.prov.GetProviderString() - // seems pointless - _, err := pr.initialCtx.GetAuthContext(prStr) - if err != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) - } - // - asyP := internaldto.NewBasicPrimitiveContext( - pr.initialCtx.GetAuthContext, - pc.GetWriter(), - pc.GetErrWriter(), - ) - return pr.executor(asyP, subPr.GetOutputBody()) - } - return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) -} - -func (pr *AsyncHTTPMonitorPrimitive) ID() int64 { - return pr.id -} - -func (pr *AsyncHTTPMonitorPrimitive) GetInputFromAlias(string) (internaldto.ExecutorOutput, bool) { - var rv internaldto.ExecutorOutput - return rv, false -} - -func (pr *AsyncHTTPMonitorPrimitive) SetExecutor(_ func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput) error { - return fmt.Errorf("AsyncHTTPMonitorPrimitive does not support SetExecutor()") -} - -func NewAsyncMonitor( - handlerCtx handler.HandlerContext, - prov provider.IProvider, - op formulation.OperationStore, - isReturning bool, -) (IAsyncMonitor, error) { - //nolint:gocritic //TODO: refactor - switch prov.GetProviderString() { - case "google": - return newGoogleAsyncMonitor(handlerCtx, prov, op, prov.GetVersion(), isReturning) - } - return nil, fmt.Errorf( - "async operation monitor for provider = '%s', api version = '%s' currently not supported", - prov.GetProviderString(), prov.GetVersion()) -} - -func newGoogleAsyncMonitor( - handlerCtx handler.HandlerContext, - prov provider.IProvider, - op formulation.OperationStore, - version string, //nolint:unparam // TODO: refactor - isReturning bool, //nolint:unparam,revive // TODO: refactor -) (IAsyncMonitor, error) { - //nolint:gocritic //TODO: refactor - switch version { - default: - return &DefaultGoogleAsyncMonitor{ - handlerCtx: handlerCtx, - prov: prov, - op: op, - }, nil - } -} - -type DefaultGoogleAsyncMonitor struct { - handlerCtx handler.HandlerContext - prov provider.IProvider - op formulation.OperationStore -} - -func (gm *DefaultGoogleAsyncMonitor) GetMonitorPrimitive( - prov provider.IProvider, - op formulation.OperationStore, - precursor primitive.IPrimitive, - initialCtx primitive.IPrimitiveCtx, - comments sqlparser.CommentDirectives, - isReturning bool, - insertCtx drm.PreparedStatementCtx, - drmCfg drm.Config, -) (primitive.IPrimitive, error) { - //nolint:gocritic,staticcheck //TODO: refactor - switch strings.ToLower(prov.GetVersion()) { - default: - return gm.getV1Monitor(prov, op, precursor, initialCtx, comments, isReturning, insertCtx, drmCfg) - } -} - -func (gm *DefaultGoogleAsyncMonitor) getV1Monitor( - prov provider.IProvider, - op formulation.OperationStore, - precursor primitive.IPrimitive, - initialCtx primitive.IPrimitiveCtx, - comments sqlparser.CommentDirectives, - isReturning bool, - insertCtx drm.PreparedStatementCtx, - drmCfg drm.Config, -) (primitive.IPrimitive, error) { - provider, providerErr := prov.GetProvider() - if providerErr != nil { - return nil, providerErr - } - ex, exPrepErr := execution.GetMonitorExecutor( - gm.handlerCtx, - provider, - op, - precursor, - initialCtx, - comments, - isReturning, - insertCtx, - drmCfg, - ) - if exPrepErr != nil { - return nil, exPrepErr - } - return ex, nil -} diff --git a/internal/stackql/bundle/bundle.go b/internal/stackql/bundle/bundle.go index f36c2f48f..534fae0f1 100644 --- a/internal/stackql/bundle/bundle.go +++ b/internal/stackql/bundle/bundle.go @@ -3,18 +3,16 @@ package bundle import ( "io" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/dbmsinternal" "github.com/stackql/stackql/internal/stackql/garbagecollector" "github.com/stackql/stackql/internal/stackql/kstore" - "github.com/stackql/stackql/internal/stackql/persistence" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/tablenamespace" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/pkg/txncounter" @@ -28,7 +26,6 @@ type Bundle interface { GetDBMSInternalRouter() dbmsinternal.Router GetSQLDataSources() map[string]sql_datasource.SQLDataSource GetSQLSystem() sql_system.SQLSystem - GetPersistenceSystem() formulation.PersistenceSystem GetSQLEngine() sqlengine.SQLEngine GetTxnCounterManager() txncounter.Manager GetTxnStore() kstore.KStore @@ -61,7 +58,6 @@ func NewBundle( namespaces: namespaces, sqlEngine: sqlEngine, sqlSystem: sqlSystem, - persistenceSystem: persistence.NewSQLPersistenceSystem(sqlSystem), controlAttributes: controlAttributes, txnStore: txnStore, txnCtrMgr: txnCtrMgr, @@ -81,7 +77,6 @@ type simpleBundle struct { namespaces tablenamespace.Collection sqlEngine sqlengine.SQLEngine sqlSystem sql_system.SQLSystem - persistenceSystem formulation.PersistenceSystem txnStore kstore.KStore txnCtrMgr txncounter.Manager typCfg typing.Config @@ -125,10 +120,6 @@ func (sb *simpleBundle) GetSQLDataSources() map[string]sql_datasource.SQLDataSou return sb.sqlDataSources } -func (sb *simpleBundle) GetPersistenceSystem() formulation.PersistenceSystem { - return sb.persistenceSystem -} - func (sb *simpleBundle) GetAuthContexts() map[string]*dto.AuthCtx { return sb.authContexts } diff --git a/internal/stackql/cmd/mcp.go b/internal/stackql/cmd/mcp.go index 0ce750552..4e9664c9a 100644 --- a/internal/stackql/cmd/mcp.go +++ b/internal/stackql/cmd/mcp.go @@ -28,16 +28,16 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" - "github.com/stackql/any-sdk/pkg/db/db_util" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/acid/tsm_physio" "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/diagnostics" "github.com/stackql/stackql/internal/stackql/entryutil" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/iqlerror" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/mcpbackend" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + db_util "github.com/stackql/stackql/internal/stackql/sqlengine/connect" "github.com/stackql/stackql/pkg/mcp_server" _ "github.com/jackc/pgx/v5" //nolint:revive // canonical driver pattern @@ -194,7 +194,7 @@ func runMCPServer(handlerCtx handler.HandlerContext, config *mcp_server.Config) cfg.DSN = dsn cfg.InitMaxRetries = 5 cfg.InitRetryInitialDelay = 2 - db, err := db_util.GetDB("pgx", "postgres", cfg) + db, err := db_util.Open("pgx", "postgres", cfg.GetDSN(), cfg.InitMaxRetries, cfg.InitRetryInitialDelay) iqlerror.PrintErrorAndExitOneIfError(err) backend, backendErr = mcpbackend.NewStackqlMCPReverseProxyService( dsn, diff --git a/internal/stackql/cmd/root.go b/internal/stackql/cmd/root.go index 536c6a972..b706d1399 100644 --- a/internal/stackql/cmd/root.go +++ b/internal/stackql/cmd/root.go @@ -22,14 +22,14 @@ import ( "runtime/debug" "strings" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/config" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/envfile" "github.com/stackql/stackql/internal/stackql/intrinsic" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/output" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/magiconair/properties" "github.com/spf13/cobra" diff --git a/internal/stackql/cmd/shell.go b/internal/stackql/cmd/shell.go index 50f03885f..4b51e7664 100644 --- a/internal/stackql/cmd/shell.go +++ b/internal/stackql/cmd/shell.go @@ -23,8 +23,6 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/config" @@ -32,8 +30,10 @@ import ( "github.com/stackql/stackql/internal/stackql/entryutil" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/iqlerror" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/presentation" "github.com/stackql/stackql/internal/stackql/provider" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/writer" "github.com/spf13/cobra" diff --git a/internal/stackql/config/config.go b/internal/stackql/config/config.go index 572c78259..5e3a433ba 100644 --- a/internal/stackql/config/config.go +++ b/internal/stackql/config/config.go @@ -5,8 +5,8 @@ import ( "path" "runtime" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) const defaultConfigCacheDir = ".stackql" diff --git a/internal/stackql/constants/constants.go b/internal/stackql/constants/constants.go new file mode 100644 index 000000000..35a6261b0 --- /dev/null +++ b/internal/stackql/constants/constants.go @@ -0,0 +1,84 @@ +package constants + +//nolint:lll +const ( + AnalyticsPrefix string = "stackql_analytics" + AuthTypeDelimiter string = "::" + AuthTypeSQLDataSourcePrefix string = "sql_data_source" + GoogleV1DiscoveryDoc string = "https://www.googleapis.com/discovery/v1/apis" + GoogleV1OperationURLPropertyString string = "selfLink" + GoogleV1ProviderCacheName string = "google_provider_v_0_3_7" + stackqlKeyTmplStr string = "__KEY_TEMPLATE__" + stackqlPathKey string = "name" + ServiceAccountRevokeErrStr string = `[INFO] Only interactive login credentials can be revoked, to authenticate with a different service account change the credentialsfilepath in the .stackqlrc file or reauthenticate with a different service account using the AUTH command` + ServiceAccountPathErrStr string = `[ERROR] credentialsfilepath not supplied or key file does not exist.` + OAuthInteractiveAuthErrStr string = `[INFO] Interactive credentials must be revoked before logging in with a different user, use the AUTH REVOKE command before attempting to authenticate again.` + NotAuthenticatedShowStr string = `[INFO] Not authenticated, use the AUTH command to authenticate to a provider.` + JSONStr string = "json" + TableStr string = "table" + CSVStr string = "csv" + TextStr string = "text" + PostgresIDMaxWidth int = 63 + PostgresJSONCastSuffix string = "::json" + PrettyTextStr string = "pptext" + DBEngineSQLite3Embedded string = "sqlite3_embedded" + DBEnginePostgresTCP string = "postgres_tcp" + DBEngineSnowflakeTCP string = "snowflake_tcp" + DBEngineDefault string = DBEngineSQLite3Embedded + SQLDialectSQLite3 string = "sqlite3" + SQLDialectPostgres string = "postgres" + SQLDialectSnowflake string = "snowflake" + SQLDbNameSnowflake string = "snowflake" + SQLDialectDefault string = SQLDialectSQLite3 + SQLFuncJSONExtractSQLite string = "json_extract" + SQLFuncJSONExtractPostgres string = "json_extract_path_text" + SQLFuncJSONExtractConformed string = SQLFuncJSONExtractSQLite + SQLFuncGroupConcatSQLite string = "group_concat" + SQLFuncGroupConcatPostgres string = "string_agg" + SQLFuncGroupConcatConformed string = SQLFuncGroupConcatSQLite + DefaulHTTPBodyFormat string = JSONStr + DefaultPrettyPrintBaseIndent int = 2 + DefaultPrettyPrintIndent int = 2 + DefaultQueryCacheSize int = 10000 + MaxDigits32BitUnsigned int = 10 + DefaultAnalyticsTemplateString string = "stackql_analytics_{{ .objectName }}" + DefaultViewsTemplateString string = "stackql_views.{{ .objectName }}" + DefaultAnalyticsRegexpString string = `^stackql_analytics_(?P.*)$` + DefaultViewsRegexpString string = `^stackql_views\.(?P.*)$` + ControlColumnCount int = 4 +) + +const ( + SQLDataSourceSchemaPostgresInfo string = "pgi" + SQLDataSourceSchemaDefault string = SQLDataSourceSchemaPostgresInfo +) + +const ( + LimitsIndirectMaxChainLength int = 1 +) + +const ( + NilTabulationName string = "__any__sdk__nil__" +) + +const ( + ReversalStreamAlias string = "reversal_stream" + ReversalStreamID int64 = -1 +) + +type GCStatus int + +const ( + GCWhite GCStatus = iota + GCBlack + GCGrey +) + +type BackendQueryType int + +const ( + BackendExec BackendQueryType = iota + BackendQuery + BackendNop + BackendTableObject +) diff --git a/internal/stackql/constants/isolation_level.go b/internal/stackql/constants/isolation_level.go new file mode 100644 index 000000000..116fa4531 --- /dev/null +++ b/internal/stackql/constants/isolation_level.go @@ -0,0 +1,18 @@ +package constants + +type IsolationLevel int + +// Isolation levels. +const ( + ReadUncommitted IsolationLevel = iota + ReadCommitted + RepeatableRead + Serializable +) + +const ( + ReadCommittedStr = "read committed" + RepeatableReadStr = "repeatable read" + SerializableStr = "serializable" + ReadUncommittedStr = "read uncommitted" +) diff --git a/internal/stackql/constants/rollback_type.go b/internal/stackql/constants/rollback_type.go new file mode 100644 index 000000000..ad7ebaa08 --- /dev/null +++ b/internal/stackql/constants/rollback_type.go @@ -0,0 +1,14 @@ +package constants + +type RollbackType int + +// Rollback algorithms. +const ( + NopRollback RollbackType = iota + EagerRollback +) + +const ( + NopRollbackStr = "nop" + EagerRollbackStr = "eager" +) diff --git a/internal/stackql/data_staging/output_data_staging/packet_preparator.go b/internal/stackql/data_staging/output_data_staging/packet_preparator.go index 956220dd8..56e24a50f 100644 --- a/internal/stackql/data_staging/output_data_staging/packet_preparator.go +++ b/internal/stackql/data_staging/output_data_staging/packet_preparator.go @@ -4,10 +4,10 @@ import ( "fmt" "github.com/lib/pq/oid" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql/internal/stackql/drm" + "github.com/stackql/stackql/internal/stackql/logging" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/streaming" "github.com/stackql/stackql/internal/stackql/typing" ) diff --git a/internal/stackql/dataflow/collection.go b/internal/stackql/dataflow/collection.go deleted file mode 100644 index 1bf9367f5..000000000 --- a/internal/stackql/dataflow/collection.go +++ /dev/null @@ -1,242 +0,0 @@ -package dataflow - -import ( - "fmt" - "sync" - - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "gonum.org/v1/gonum/graph" - "gonum.org/v1/gonum/graph/simple" - "gonum.org/v1/gonum/graph/topo" -) - -type Unit interface { - iDataFlowUnit() -} - -// Collection is the DAG representing. -type Collection interface { - AddOrUpdateEdge( - source Vertex, - dest Vertex, - comparisonExpr *sqlparser.ComparisonExpr, - sourceExpr sqlparser.Expr, - destColumn *sqlparser.ColName) error - AddVertex(v Vertex) - GetAllUnits() ([]Unit, error) - GetNextID() int64 - InDegree(v Vertex) int - OutDegree(v Vertex) int - Sort() error - Vertices() []Vertex - UpsertStandardDataFlowVertex(taxonomy.AnnotationCtx, sqlparser.TableExpr) Vertex - getVertex(annotation taxonomy.AnnotationCtx, tableExpr sqlparser.TableExpr) (Vertex, bool) - GetSplitAnnotationContextMap() taxonomy.AnnotationCtxSplitMap - WithSplitAnnotationContextMap(taxonomy.AnnotationCtxSplitMap) Collection -} - -func NewStandardDataFlowCollection(cfg dto.DataFlowCfg) Collection { - return &standardDataFlowCollection{ - idMutex: &sync.Mutex{}, - g: simple.NewWeightedDirectedGraph(0.0, 0.0), - vertices: make(map[Vertex]struct{}), - verticesForTableExprs: make(map[sqlparser.TableExpr]struct{}), - splitAnnotationContextMap: taxonomy.NewAnnotationCtxSplitMap(), - cfg: cfg, - } -} - -// Upsert a vertex into the collection. -// If the vertex already exists, it is returned. -// Otherwise, a new vertex is created and returned. -// This makes subsequent handling of degenerate edges easier. -func (dc *standardDataFlowCollection) UpsertStandardDataFlowVertex( - annotation taxonomy.AnnotationCtx, - tableExpr sqlparser.TableExpr, -) Vertex { - if v, ok := dc.getVertex(annotation, tableExpr); ok { - return v - } - return &standardDataFlowVertex{ - annotation: annotation, - tableExpr: tableExpr, - id: dc.GetNextID(), - } -} - -func (dc *standardDataFlowCollection) getVertex( - annotation taxonomy.AnnotationCtx, - tableExpr sqlparser.TableExpr, -) (Vertex, bool) { - for v := range dc.vertices { - rhsAnnotation := v.GetAnnotation() - rhsTableExpr := v.GetTableExpr() - annotationMatch := rhsAnnotation.Equals(annotation) - tableExprMatch := rhsTableExpr == tableExpr - if annotationMatch && tableExprMatch { - return v, true - } - if annotationMatch || tableExprMatch { - logging.GetLogger().Infof( - "data flow error: vertex with annotation '%v' and tableExpr '%v' has **almost** been found", - rhsAnnotation, - rhsTableExpr, - ) - } - } - return nil, false -} - -type standardDataFlowCollection struct { - idMutex *sync.Mutex - maxID int64 - g *simple.WeightedDirectedGraph - sorted []graph.Node - orphans []Vertex - weaklyConnnectedGraphs []WeaklyConnectedComponent - vertices map[Vertex]struct{} - verticesForTableExprs map[sqlparser.TableExpr]struct{} - edges []Edge - splitAnnotationContextMap taxonomy.AnnotationCtxSplitMap - cfg dto.DataFlowCfg -} - -func (dc *standardDataFlowCollection) GetNextID() int64 { - defer dc.idMutex.Unlock() - dc.idMutex.Lock() - dc.maxID++ - return dc.maxID -} - -func (dc *standardDataFlowCollection) GetSplitAnnotationContextMap() taxonomy.AnnotationCtxSplitMap { - return dc.splitAnnotationContextMap -} - -func (dc *standardDataFlowCollection) WithSplitAnnotationContextMap(m taxonomy.AnnotationCtxSplitMap) Collection { - dc.splitAnnotationContextMap = m - return dc -} - -func (dc *standardDataFlowCollection) AddOrUpdateEdge( - source Vertex, - dest Vertex, - comparisonExpr *sqlparser.ComparisonExpr, - sourceExpr sqlparser.Expr, - destColumn *sqlparser.ColName, -) error { - dc.AddVertex(source) - dc.AddVertex(dest) - sourceID := source.ID() - destID := dest.ID() - existingEdge := dc.g.WeightedEdge(sourceID, destID) - if existingEdge == nil { - edge := newStandardDataFlowEdge(source, dest, comparisonExpr, sourceExpr, destColumn) - dc.edges = append(dc.edges, edge) - dc.g.SetWeightedEdge(edge) - return nil - } - switch existingEdge := existingEdge.(type) { - case Edge: - existingEdge.AddRelation(NewStandardDataFlowRelation(comparisonExpr, destColumn, sourceExpr)) - default: - return fmt.Errorf("cannnot accomodate data flow edge of type: '%T'", existingEdge) - } - return nil -} - -func (dc *standardDataFlowCollection) AddVertex(v Vertex) { - _, ok := dc.verticesForTableExprs[v.GetTableExpr()] - if ok { - var isNew bool - if v.GetEquivalencyGroup() > 0 { - _, isNew = dc.g.NodeWithID(v.ID()) // TODO: change to acceptable idion - } - if !isNew { - return - } - } - dc.vertices[v] = struct{}{} - dc.verticesForTableExprs[v.GetTableExpr()] = struct{}{} - dc.g.AddNode(v) -} - -// Sort sorts the underlying graph. -// Cyclic dependencies return an error. -// This is where the DAG invariant is enforced. -func (dc *standardDataFlowCollection) Sort() error { - var err error - dc.sorted, err = topo.Sort(dc.g) - if err != nil { - return err - } - err = dc.optimise() - return err -} - -func (dc *standardDataFlowCollection) Vertices() []Vertex { - var rv []Vertex - for vert := range dc.vertices { - rv = append(rv, vert) - } - return rv -} - -func (dc *standardDataFlowCollection) GetAllUnits() ([]Unit, error) { - var rv []Unit - for _, orphan := range dc.orphans { - rv = append(rv, orphan) - } - for _, component := range dc.weaklyConnnectedGraphs { - rv = append(rv, component) - } - return rv, nil -} - -func (dc *standardDataFlowCollection) InDegree(v Vertex) int { - inDegree := 0 - for _, e := range dc.edges { - if e.GetDest() == v { - inDegree++ - } - } - return inDegree -} - -func (dc *standardDataFlowCollection) OutDegree(v Vertex) int { - outDegree := 0 - for _, e := range dc.edges { - if e.GetSource() == v { - outDegree++ - } - } - return outDegree -} - -func (dc *standardDataFlowCollection) optimise() error { - for _, node := range dc.sorted { - switch node := node.(type) { - case Vertex: - logging.GetLogger().Debugf("%v\n", node) - inDegree := dc.g.To(node.ID()).Len() - outDegree := dc.g.From(node.ID()).Len() - if inDegree == 0 && outDegree == 0 { - dc.orphans = append(dc.orphans, node) - continue - } - if inDegree == 0 && outDegree != 0 { - component := NewStandardDataFlowWeaklyConnectedComponent(dc, node) - err := component.Analyze() - if err != nil { - return err - } - dc.weaklyConnnectedGraphs = append(dc.weaklyConnnectedGraphs, component) - } - default: - return fmt.Errorf("cannot accomodate dataflow element of type = '%t'", node) - } - } - return nil -} diff --git a/internal/stackql/dataflow/component.go b/internal/stackql/dataflow/component.go deleted file mode 100644 index 8ec55ba53..000000000 --- a/internal/stackql/dataflow/component.go +++ /dev/null @@ -1,135 +0,0 @@ -package dataflow - -import ( - "fmt" - - "gonum.org/v1/gonum/graph" -) - -type WeaklyConnectedComponent interface { - Unit - AddEdge(Edge) - Analyze() error - GetEdges() ([]Edge, error) - GetOrderedNodes() ([]Vertex, error) - PushBack(Vertex) -} - -type standardDataFlowWeaklyConnectedComponent struct { - idsVisited map[int64]struct{} - collection *standardDataFlowCollection - root graph.Node - orderedNodes []graph.Node - edges []graph.Edge -} - -func NewStandardDataFlowWeaklyConnectedComponent( - collection *standardDataFlowCollection, - root graph.Node, -) WeaklyConnectedComponent { - return &standardDataFlowWeaklyConnectedComponent{ - collection: collection, - root: root, - idsVisited: map[int64]struct{}{ - root.ID(): {}, - }, - orderedNodes: []graph.Node{ - root, - }, - } -} - -func (wc *standardDataFlowWeaklyConnectedComponent) GetOrderedNodes() ([]Vertex, error) { - var rv []Vertex - for _, n := range wc.orderedNodes { - switch n := n.(type) { - case Vertex: - rv = append(rv, n) - default: - return nil, fmt.Errorf("data flow error: weakly connected components cannot accomodate nodes of type '%T'", n) - } - } - return rv, nil -} - -func (wc *standardDataFlowWeaklyConnectedComponent) GetEdges() ([]Edge, error) { - var rv []Edge - for _, n := range wc.edges { - switch n := n.(type) { - case Edge: - rv = append(rv, n) - default: - return nil, fmt.Errorf("data flow error: weakly connected components cannot accomodate edges of type '%T'", n) - } - } - return rv, nil -} - -func (wc *standardDataFlowWeaklyConnectedComponent) isAlreadyInOrderedNodes(rhs graph.Node) bool { - for _, n := range wc.orderedNodes { - if n.ID() == rhs.ID() { - return true - } - } - return false -} - -func (wc *standardDataFlowWeaklyConnectedComponent) Analyze() error { - // This algorithm underlying this analyisis - // is defective; or would be were it not halted early. - // TODO: Upgrade this algorithm and make limits configurable - // and document the algorithm. - for _, node := range wc.collection.sorted { - incidentNodes := wc.collection.g.To(node.ID()) - wc.idsVisited[node.ID()] = struct{}{} - if incidentNodes.Len() > wc.collection.cfg.GetMaxDependencies() { - return fmt.Errorf( - "data flow: too complex for now; %d dependencies detected when max allowed = %d", - incidentNodes.Len(), - wc.collection.cfg.GetMaxDependencies(), - ) - } - for { - itemPresent := incidentNodes.Next() - if !itemPresent { - // wc.orderedNodes = append(wc.orderedNodes, node) - // wc.idsVisited[node.ID()] = struct{}{} - break - } - fromNode := incidentNodes.Node() - _, ok := wc.idsVisited[fromNode.ID()] - if ok { - isAlreadyInOrderedNodes := wc.isAlreadyInOrderedNodes(node) - if !isAlreadyInOrderedNodes { - wc.orderedNodes = append(wc.orderedNodes, node) - } - wc.idsVisited[node.ID()] = struct{}{} - incidentEdge := wc.collection.g.WeightedEdge(fromNode.ID(), node.ID()) - if incidentEdge == nil { - return fmt.Errorf("found nil edge in data flow graph") - } - wc.edges = append(wc.edges, incidentEdge) - } else { - // TODO: improve error, or obviate with superior algorithm - return fmt.Errorf( - "data flow: error: complexity not yet supported; only single data flow dependencies allowed at present", - ) - } - } - } - return nil -} - -func (wc *standardDataFlowWeaklyConnectedComponent) AddEdge(e Edge) { - wc.edges = append(wc.edges, e) -} - -func (wc *standardDataFlowWeaklyConnectedComponent) PushBack(v Vertex) { - isAlreadyInOrderedNodes := wc.isAlreadyInOrderedNodes(v) - if isAlreadyInOrderedNodes { - return - } - wc.orderedNodes = append(wc.orderedNodes, v) -} - -func (wc *standardDataFlowWeaklyConnectedComponent) iDataFlowUnit() {} diff --git a/internal/stackql/dataflow/edge.go b/internal/stackql/dataflow/edge.go deleted file mode 100644 index 379782478..000000000 --- a/internal/stackql/dataflow/edge.go +++ /dev/null @@ -1,130 +0,0 @@ -package dataflow - -import ( - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "gonum.org/v1/gonum/graph" -) - -type Edge interface { - graph.WeightedEdge - AddRelation(Relation) - GetColumnDescriptors() ([]formulation.ColumnDescriptor, error) - GetDest() Vertex - GetProjection() (map[string]string, error) - GetSelectExprs() (sqlparser.SelectExprs, error) - HasSourceAttribute(attr string) bool - GetSource() Vertex - IsSQL() bool -} - -type standardDataFlowEdge struct { - source, dest Vertex - relations []Relation -} - -func newStandardDataFlowEdge( - source Vertex, - dest Vertex, - comparisonExpr *sqlparser.ComparisonExpr, - sourceExpr sqlparser.Expr, - destColumn *sqlparser.ColName, -) Edge { - return &standardDataFlowEdge{ - source: source, - dest: dest, - relations: []Relation{ - NewStandardDataFlowRelation( - comparisonExpr, - destColumn, - sourceExpr, - ), - }, - } -} - -func (de *standardDataFlowEdge) AddRelation(rel Relation) { - de.relations = append(de.relations, rel) -} - -func (de *standardDataFlowEdge) From() graph.Node { - return de.source -} - -func (de *standardDataFlowEdge) To() graph.Node { - return de.dest -} - -func (de *standardDataFlowEdge) ReversedEdge() graph.Edge { - // Reversal is invalid given the assymetric - // expressions, therefore returning unaltered - // as per library recommmendation. - return de -} - -func (de *standardDataFlowEdge) Weight() float64 { - return 1.0 -} - -func (de *standardDataFlowEdge) GetSource() Vertex { - return de.source -} - -func (de *standardDataFlowEdge) IsSQL() bool { - for _, rel := range de.relations { - if rel.IsSQL() { - return true - } - } - return false -} - -func (de *standardDataFlowEdge) GetDest() Vertex { - return de.dest -} - -func (de *standardDataFlowEdge) GetProjection() (map[string]string, error) { - rv := make(map[string]string) - for _, rel := range de.relations { - src, dst, err := rel.GetProjection() - if err != nil { - return nil, err - } - rv[src] = dst - } - return rv, nil -} - -func (de *standardDataFlowEdge) GetSelectExprs() (sqlparser.SelectExprs, error) { - var rv sqlparser.SelectExprs - for _, rel := range de.relations { - selExpr, err := rel.GetSelectExpr() - if err != nil { - return nil, err - } - rv = append(rv, selExpr) - } - return rv, nil -} - -func (de *standardDataFlowEdge) HasSourceAttribute(attr string) bool { - for _, rel := range de.relations { - dest := rel.GetDestColumnName() - if dest == attr { - return true - } - } - return false -} - -func (de *standardDataFlowEdge) GetColumnDescriptors() ([]formulation.ColumnDescriptor, error) { - var rv []formulation.ColumnDescriptor - for _, rel := range de.relations { - d, err := rel.GetColumnDescriptor() - if err != nil { - return nil, err - } - rv = append(rv, d) - } - return rv, nil -} diff --git a/internal/stackql/dataflow/relation.go b/internal/stackql/dataflow/relation.go deleted file mode 100644 index 77f12ffe1..000000000 --- a/internal/stackql/dataflow/relation.go +++ /dev/null @@ -1,73 +0,0 @@ -package dataflow - -import ( - "fmt" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/pkg/logging" -) - -type Relation interface { - GetProjection() (string, string, error) - GetSelectExpr() (sqlparser.SelectExpr, error) - GetColumnDescriptor() (formulation.ColumnDescriptor, error) - GetDestColumnName() string - IsSQL() bool -} - -type standardDataFlowRelation struct { - comparisonExpr *sqlparser.ComparisonExpr - destColumn *sqlparser.ColName - sourceExpr sqlparser.Expr -} - -func NewStandardDataFlowRelation( - comparisonExpr *sqlparser.ComparisonExpr, - destColumn *sqlparser.ColName, - sourceExpr sqlparser.Expr, -) Relation { - return &standardDataFlowRelation{ - comparisonExpr: comparisonExpr, - destColumn: destColumn, - sourceExpr: sourceExpr, - } -} - -func (dr *standardDataFlowRelation) GetDestColumnName() string { - return dr.destColumn.Name.GetRawVal() -} - -func (dr *standardDataFlowRelation) GetProjection() (string, string, error) { - switch se := dr.sourceExpr.(type) { - case *sqlparser.ColName: - return se.Name.GetRawVal(), dr.destColumn.Name.GetRawVal(), nil - default: - return "", "", fmt.Errorf("cannot project from expression type = '%T'", se) - } -} - -func (dr *standardDataFlowRelation) IsSQL() bool { - switch se := dr.sourceExpr.(type) { - case *sqlparser.ColName: - return false - default: - logging.GetLogger().Infof("%v\n", se) - return true - } -} - -func (dr *standardDataFlowRelation) GetSelectExpr() (sqlparser.SelectExpr, error) { - rv := &sqlparser.AliasedExpr{ - Expr: dr.sourceExpr, - As: dr.destColumn.Name, - } - return rv, nil -} - -func (dr *standardDataFlowRelation) GetColumnDescriptor() (formulation.ColumnDescriptor, error) { - decoratedColumn := fmt.Sprintf(`%s AS %s`, sqlparser.ColDelimitedString(dr.sourceExpr), dr.destColumn.Name.GetRawVal()) - cd := formulation.NewColumnDescriptor(dr.destColumn.Name.GetRawVal(), "", "", decoratedColumn, nil, nil, nil) - return cd, nil -} diff --git a/internal/stackql/dataflow/vertex.go b/internal/stackql/dataflow/vertex.go deleted file mode 100644 index 98b2892dc..000000000 --- a/internal/stackql/dataflow/vertex.go +++ /dev/null @@ -1,45 +0,0 @@ -package dataflow - -import ( - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "gonum.org/v1/gonum/graph" -) - -type Vertex interface { - graph.Node - Unit - GetAnnotation() taxonomy.AnnotationCtx - GetEquivalencyGroup() int64 - SetEquivalencyGroup(id int64) - GetTableExpr() sqlparser.TableExpr -} - -type standardDataFlowVertex struct { - id int64 - equiValencyGroupID int64 - annotation taxonomy.AnnotationCtx - tableExpr sqlparser.TableExpr -} - -func (dv *standardDataFlowVertex) iDataFlowUnit() {} - -func (dv *standardDataFlowVertex) ID() int64 { - return dv.id -} - -func (dv *standardDataFlowVertex) GetEquivalencyGroup() int64 { - return dv.equiValencyGroupID -} - -func (dv *standardDataFlowVertex) SetEquivalencyGroup(id int64) { - dv.equiValencyGroupID = id -} - -func (dv *standardDataFlowVertex) GetAnnotation() taxonomy.AnnotationCtx { - return dv.annotation -} - -func (dv *standardDataFlowVertex) GetTableExpr() sqlparser.TableExpr { - return dv.tableExpr -} diff --git a/internal/stackql/datasource/sql_datasource/generic.go b/internal/stackql/datasource/sql_datasource/generic.go index bb6e9801d..0be819e8b 100644 --- a/internal/stackql/datasource/sql_datasource/generic.go +++ b/internal/stackql/datasource/sql_datasource/generic.go @@ -8,17 +8,21 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" //nolint:revive,nolintlint // this is a DB driver pattern _ "github.com/snowflakedb/gosnowflake" //nolint:revive,nolintlint // this is a DB driver pattern - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/db_util" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/datasource/sqltable" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + db_util "github.com/stackql/stackql/internal/stackql/sqlengine/connect" ) var ( _ SQLDataSource = &genericSQLDataSource{} - getDBFunc = db_util.GetDB //nolint:gochecknoglobals // patching variable + getDBFunc = openDB //nolint:gochecknoglobals // patching variable ) +func openDB(driverName, dbName string, cfg dto.SQLBackendCfg) (*sql.DB, error) { + return db_util.Open(driverName, dbName, cfg.GetDSN(), cfg.InitMaxRetries, cfg.InitRetryInitialDelay) +} + func NewGenericSQLDataSource(authCtx *dto.AuthCtx, driverName string, dbName string) (SQLDataSource, error) { sqlCfg, hasSQLCfg := authCtx.GetSQLCfg() if !hasSQLCfg { diff --git a/internal/stackql/datasource/sql_datasource/generic_test.go b/internal/stackql/datasource/sql_datasource/generic_test.go index e252b3f3f..f1a551901 100644 --- a/internal/stackql/datasource/sql_datasource/generic_test.go +++ b/internal/stackql/datasource/sql_datasource/generic_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/internal/stackql/datasource/sql_datasource/sql_datasource.go b/internal/stackql/datasource/sql_datasource/sql_datasource.go index 95cc6159d..807359418 100644 --- a/internal/stackql/datasource/sql_datasource/sql_datasource.go +++ b/internal/stackql/datasource/sql_datasource/sql_datasource.go @@ -5,9 +5,9 @@ import ( "database/sql" "fmt" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/datasource/sqltable" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) // ContextQuerier is the optional cancellation-capable query handle used by read adapters. diff --git a/internal/stackql/datasource/sql_datasource/sql_datasource_test.go b/internal/stackql/datasource/sql_datasource/sql_datasource_test.go index a7286c906..1789d56ac 100644 --- a/internal/stackql/datasource/sql_datasource/sql_datasource_test.go +++ b/internal/stackql/datasource/sql_datasource/sql_datasource_test.go @@ -5,9 +5,10 @@ import ( "fmt" "testing" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/datasource/sqltable" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -65,7 +66,7 @@ func TestNewDataSource(t *testing.T) { }) t.Run("authCtx is not supported", func(t *testing.T) { - authCtx := &dto.AuthCtx{Type: "not_supported"} + authCtx := &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: "not_supported"}} ds, err := NewDataSource(authCtx, nil) assert.Error(t, err) assert.Nil(t, ds) @@ -73,24 +74,24 @@ func TestNewDataSource(t *testing.T) { }) t.Run("authCtx.Type is snowflake", func(t *testing.T) { - authCtx := &dto.AuthCtx{Type: fmt.Sprintf( + authCtx := &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: fmt.Sprintf( "%s%s%s", constants.AuthTypeSQLDataSourcePrefix, constants.AuthTypeDelimiter, "snowflake", - )} + )}} ds, err := NewDataSource(authCtx, genericSQLDataSourceFuncMock) assert.NotNil(t, ds) assert.Nil(t, err) }) t.Run("authCtx.Type is postgres", func(t *testing.T) { - authCtx := &dto.AuthCtx{Type: fmt.Sprintf( + authCtx := &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: fmt.Sprintf( "%s%s%s", constants.AuthTypeSQLDataSourcePrefix, constants.AuthTypeDelimiter, "postgres", - )} + )}} ds, err := NewDataSource(authCtx, genericSQLDataSourceFuncMock) assert.NotNil(t, ds) assert.Nil(t, err) diff --git a/internal/stackql/dbmsinternal/dbmsinternal.go b/internal/stackql/dbmsinternal/dbmsinternal.go index 242f8b743..8afa4d618 100644 --- a/internal/stackql/dbmsinternal/dbmsinternal.go +++ b/internal/stackql/dbmsinternal/dbmsinternal.go @@ -4,10 +4,10 @@ import ( "regexp" "strings" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/stackql/stackql/internal/stackql/logging" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" ) diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go deleted file mode 100644 index df46aa8ce..000000000 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ /dev/null @@ -1,1140 +0,0 @@ -package dependencyplanner - -import ( - "fmt" - "strings" - - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/media" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/astindirect" - "github.com/stackql/stackql/internal/stackql/astvisit" - "github.com/stackql/stackql/internal/stackql/dataflow" - "github.com/stackql/stackql/internal/stackql/docparser" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/primitivebuilder" - "github.com/stackql/stackql/internal/stackql/primitivecomposer" - "github.com/stackql/stackql/internal/stackql/pushdown" - "github.com/stackql/stackql/internal/stackql/sqlrewrite" - "github.com/stackql/stackql/internal/stackql/sqlstream" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/internal/stackql/util" - "github.com/stackql/stackql/pkg/astformat" -) - -type DependencyPlanner interface { - Plan() error - GetBldr() primitivebuilder.Builder - GetSelectCtx() drm.PreparedStatementCtx - WithPrepStmtOffset(offset int) DependencyPlanner - WithElideRead(isElideRead bool) DependencyPlanner -} - -type standardDependencyPlanner struct { - annotatedAST annotatedast.AnnotatedAst - dataflowCollection dataflow.Collection - colRefs parserutil.ColTableMap - handlerCtx handler.HandlerContext - execSlice []primitivebuilder.Builder - primaryTcc, tcc internaldto.TxnControlCounters - primitiveComposer primitivecomposer.PrimitiveComposer - rewrittenWhere *sqlparser.Where - secondaryTccs []internaldto.TxnControlCounters - sqlStatement *sqlparser.Select - tableSlice []tableinsertioncontainer.TableInsertionContainer - tblz taxonomy.TblMap - discoGenIDs map[sqlparser.SQLNode]int - tccSetAheadOfTime bool - - // - bldr primitivebuilder.Builder - selCtx drm.PreparedStatementCtx - defaultStream streaming.MapStream - annMap taxonomy.AnnotationCtxMap - equivalencyGroupTCCs map[int64]internaldto.TxnControlCounters - // - prepStmtOffset int - // - isElideRead bool - // - dataflowToEdges map[int][]int - nodeIDIdxMap map[int64]int -} - -func NewStandardDependencyPlanner( - annotatedAST annotatedast.AnnotatedAst, - handlerCtx handler.HandlerContext, - dataflowCollection dataflow.Collection, - colRefs parserutil.ColTableMap, - rewrittenWhere *sqlparser.Where, - sqlStatement *sqlparser.Select, - tblz taxonomy.TblMap, - primitiveComposer primitivecomposer.PrimitiveComposer, - tcc internaldto.TxnControlCounters, - tccSetAheadOfTime bool, -) (DependencyPlanner, error) { - if tcc == nil { - return nil, fmt.Errorf("violation of standardDependencyPlanner invariant: txn counter cannot be nil") - } - return &standardDependencyPlanner{ - annotatedAST: annotatedAST, - handlerCtx: handlerCtx, - dataflowCollection: dataflowCollection, - colRefs: colRefs, - rewrittenWhere: rewrittenWhere, - sqlStatement: sqlStatement, - tblz: tblz, - primitiveComposer: primitiveComposer, - discoGenIDs: make(map[sqlparser.SQLNode]int), - defaultStream: streaming.NewStandardMapStream(), - annMap: make(taxonomy.AnnotationCtxMap), - tcc: tcc, - tccSetAheadOfTime: tccSetAheadOfTime, - equivalencyGroupTCCs: make(map[int64]internaldto.TxnControlCounters), - dataflowToEdges: make(map[int][]int), - nodeIDIdxMap: make(map[int64]int), - }, nil -} - -func isAnnotationIndirection(annotationCtx taxonomy.AnnotationCtx) bool { - _, isView := annotationCtx.GetView() - _, isSubquery := annotationCtx.GetSubquery() - return isView || isSubquery -} - -// orchestrateIndirection handles indirection vertices (views, subqueries, CTEs) in the dependency planner. -// Unlike regular tables that need HTTP acquisition, indirections are already materialized by the -// "create tail" builder and appear in the FROM clause as inline subqueries. -// This method creates a NopBuilder as a placeholder in the execution DAG. -// Indirections are NOT added to dp.tableSlice because they appear as inline subqueries -// with their own internal control columns; adding them would generate incorrect outer WHERE clauses. -// -//nolint:unparam,revive // acceptable -func (dp *standardDependencyPlanner) orchestrateIndirection( - annotationCtx taxonomy.AnnotationCtx, -) (int, error) { - builder := primitivebuilder.NewNopBuilder( - dp.primitiveComposer.GetGraphHolder(), - dp.primitiveComposer.GetTxnCtrlCtrs(), - dp.handlerCtx, - dp.handlerCtx.GetSQLEngine(), - []string{}, - ) - dp.execSlice = append(dp.execSlice, builder) - idx := len(dp.execSlice) - 1 - return idx, nil -} - -func (dp *standardDependencyPlanner) dataflowEdgeExists(from, to int) bool { - edges, ok := dp.dataflowToEdges[to] - if !ok { - return false - } - for _, e := range edges { - if e == from { - return true - } - } - return false -} - -func (dp *standardDependencyPlanner) WithPrepStmtOffset(offset int) DependencyPlanner { - dp.prepStmtOffset = offset - return dp -} - -func (dp *standardDependencyPlanner) WithElideRead(isElideRead bool) DependencyPlanner { - dp.isElideRead = isElideRead - return dp -} - -func (dp *standardDependencyPlanner) GetBldr() primitivebuilder.Builder { - return dp.bldr -} - -func (dp *standardDependencyPlanner) GetSelectCtx() drm.PreparedStatementCtx { - return dp.selCtx -} - -//nolint:funlen,gocognit,gocyclo,cyclop // inherently complex function -func (dp *standardDependencyPlanner) Plan() error { - err := dp.dataflowCollection.Sort() - if err != nil { - return err - } - units, err := dp.dataflowCollection.GetAllUnits() - if err != nil { - return err - } - // TODO: lift this restriction once all traversal algorithms are adequate - weaklyConnectedComponentCount := 0 - for _, u := range units { - unit := u - switch unit := unit.(type) { - case dataflow.Vertex: - inDegree := dp.dataflowCollection.InDegree(unit) - outDegree := dp.dataflowCollection.OutDegree(unit) - if inDegree == 0 && outDegree > 0 { - // TODO: start builder - logging.GetLogger().Infof("\n") - } - if inDegree != 0 || outDegree != 0 { - return fmt.Errorf( - "cannot currently execute data dependent tables with inDegree = %d and/or outDegree = %d", - inDegree, outDegree) - } - tableExpr := unit.GetTableExpr() - annotation := unit.GetAnnotation() - _, isView := annotation.GetView() - _, isSubquery := annotation.GetSubquery() - // Note: CTEs are converted to subqueries at AST level, so isSubquery handles them. - if isView || isSubquery { - dp.annMap[tableExpr] = annotation - continue - } - dp.annMap[tableExpr] = annotation - connectorStream := streaming.NewStandardMapStream() - insPsc, _, insErr := dp.processOrphan(tableExpr, annotation, unit) - if insErr != nil { - return insErr - } - stream := streaming.NewNopMapStream() - idx, orcErr := dp.orchestrate(unit.GetEquivalencyGroup(), annotation, insPsc, connectorStream, stream) - if orcErr != nil { - return orcErr - } - dp.nodeIDIdxMap[unit.ID()] = idx - case dataflow.WeaklyConnectedComponent: - weaklyConnectedComponentCount++ - orderedNodes, oErr := unit.GetOrderedNodes() - if oErr != nil { - return oErr - } - dp.nodeIDIdxMap = make(map[int64]int) - logging.GetLogger().Infof("%v\n", orderedNodes) - edges, eErr := unit.GetEdges() - if eErr != nil { - return eErr - } - logging.GetLogger().Infof("%v\n", edges) - edgeCount := len(edges) - // TODO: test this - dependencyMax := dp.handlerCtx.GetRuntimeContext().DataflowDependencyMax - if edgeCount > dependencyMax { - return fmt.Errorf( - "data flow: cannot accomodate table dependencies of this complexity: supplied = %d, max = %d", - edgeCount, dependencyMax) - } - idsVisited := make(map[int64]struct{}) - indirectionNodeIDs := make(map[int64]struct{}) - // first pass: set up AOT stuff - // - stream per edge. - edgeStreams := make(map[dataflow.Edge]streaming.MapStream) - nodeStreamCollections := NewStreamDependecyCollection() - insertPrepearedStatements := make(map[int64]drm.PreparedStatementCtx) - var orderedEdges []dataflow.Edge - for _, n := range orderedNodes { - if _, ok := idsVisited[n.ID()]; ok { - continue - } - idsVisited[n.ID()] = struct{}{} - tableExpr := n.GetTableExpr() - annotation := n.GetAnnotation() - dp.annMap[tableExpr] = annotation - // Indirection nodes (views, subqueries, CTEs) are already materialized - // by the create tail builder; register them and skip acquisition. - if isAnnotationIndirection(annotation) { - indirectionNodeIDs[n.ID()] = struct{}{} - idx, indErr := dp.orchestrateIndirection(annotation) - if indErr != nil { - return indErr - } - dp.nodeIDIdxMap[n.ID()] = idx - continue - } - for _, e := range edges { - //nolint:nestif // necessary to handle indirection cases cleanly - if e.From().ID() == n.ID() { - insPsc, tcc, insErr := dp.processOrphan(tableExpr, annotation, n) - if insErr != nil { - return insErr - } - insertPrepearedStatements[n.ID()] = insPsc - toNode := e.GetDest() - toAnnotation := toNode.GetAnnotation().Clone() // this bodge protects split source vertices - toTableExpr := toNode.GetTableExpr() - // Handle indirection destination nodes. - if isAnnotationIndirection(toAnnotation) { - if _, alreadyHandled := indirectionNodeIDs[toNode.ID()]; !alreadyHandled { - indirectionNodeIDs[toNode.ID()] = struct{}{} - dp.annMap[toTableExpr] = toAnnotation - toIdx, toIndErr := dp.orchestrateIndirection(toAnnotation) - if toIndErr != nil { - return toIndErr - } - dp.nodeIDIdxMap[toNode.ID()] = toIdx - } - orderedEdges = append(orderedEdges, e) - continue - } - stream, streamErr := dp.getStreamFromEdge(e, toAnnotation, tcc) - if streamErr != nil { - return streamErr - } - edgeStreams[e] = stream - nodeStreamCollections.Add(n.ID(), toNode.ID(), stream) - toInsPsc, _, toErr := dp.processOrphan(toTableExpr, toAnnotation, toNode) - if toErr != nil { - return toErr - } - insertPrepearedStatements[toNode.ID()] = toInsPsc - orderedEdges = append(orderedEdges, e) - } - } - } - // second pass: connect streams - // - edge collection per node - for _, e := range orderedEdges { - fromNode := e.GetSource() - toNode := e.GetDest() - fromAnnotation := fromNode.GetAnnotation() - toAnnotation := toNode.GetAnnotation().Clone() // this bodge protects split source vertices - toTableExpr := toNode.GetTableExpr() - // For indirection nodes, builders are already created; just wire up edge dependencies. - _, fromIsIndirection := indirectionNodeIDs[fromNode.ID()] - _, toIsIndirection := indirectionNodeIDs[toNode.ID()] - if fromIsIndirection && toIsIndirection { - // Both sides are indirections; no acquisition or streaming needed. - // Just ensure edge dependencies are registered. - fromIdx := dp.nodeIDIdxMap[fromNode.ID()] - toIdx := dp.nodeIDIdxMap[toNode.ID()] - if !dp.dataflowEdgeExists(fromIdx, toIdx) { - dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) - } - continue - } - //nolint:nestif // necessary to handle indirection cases cleanly - if fromIsIndirection { - // Source is indirection, destination is a regular table. - fromIdx := dp.nodeIDIdxMap[fromNode.ID()] - toIdx, toBuilderExists := dp.nodeIDIdxMap[toNode.ID()] - if !toBuilderExists { - toInsPsc, pscExists := insertPrepearedStatements[toNode.ID()] - if !pscExists { - return fmt.Errorf("unknown insert prepared statement") - } - dp.annMap[toTableExpr] = toAnnotation - toAnnotation.SetDynamic() - arrivingDestinationNodeStream := nodeStreamCollections.GetArriving(toNode.ID()) - departingDestinationNodeStream := nodeStreamCollections.GetDeparting(toNode.ID()) - var toErr error - toIdx, toErr = dp.orchestrate( - -1, toAnnotation, toInsPsc, arrivingDestinationNodeStream, departingDestinationNodeStream) - if toErr != nil { - return toErr - } - dp.nodeIDIdxMap[toNode.ID()] = toIdx - } - if !dp.dataflowEdgeExists(fromIdx, toIdx) { - dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) - } - continue - } - //nolint:nestif // necessary to handle indirection cases cleanly - if toIsIndirection { - // Destination is indirection, source is a regular table. - toIdx := dp.nodeIDIdxMap[toNode.ID()] - fromIdx, fromBuilderExists := dp.nodeIDIdxMap[fromNode.ID()] - if !fromBuilderExists { - insPsc, pscExists := insertPrepearedStatements[fromNode.ID()] - if !pscExists { - return fmt.Errorf("unknown insert prepared statement") - } - arrivingSourceNodeStream := nodeStreamCollections.GetArriving(fromNode.ID()) - departingSourceNodeStream := nodeStreamCollections.GetDeparting(fromNode.ID()) - var fromErr error - fromIdx, fromErr = dp.orchestrate(-1, fromAnnotation, insPsc, arrivingSourceNodeStream, departingSourceNodeStream) - if fromErr != nil { - return fromErr - } - dp.nodeIDIdxMap[fromNode.ID()] = fromIdx - } - if !dp.dataflowEdgeExists(fromIdx, toIdx) { - dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) - } - continue - } - // Neither side is an indirection; original logic. - departingSourceNodeStream := nodeStreamCollections.GetDeparting(fromNode.ID()) - arrivingDestinationNodeStream := nodeStreamCollections.GetArriving(toNode.ID()) - arrivingSourceNodeStream := nodeStreamCollections.GetArriving(fromNode.ID()) - departingDestinationNodeStream := nodeStreamCollections.GetDeparting(toNode.ID()) - insPsc, pscExists := insertPrepearedStatements[fromNode.ID()] - if !pscExists { - return fmt.Errorf("unknown insert prepared statement") - } - toInsPsc, pscExists := insertPrepearedStatements[toNode.ID()] - if !pscExists { - return fmt.Errorf("unknown insert prepared statement") - } - fromIdx, fromBuilderExists := dp.nodeIDIdxMap[fromNode.ID()] - if !fromBuilderExists { - var fromErr error - fromIdx, fromErr = dp.orchestrate(-1, fromAnnotation, insPsc, arrivingSourceNodeStream, departingSourceNodeStream) - if fromErr != nil { - return fromErr - } - dp.nodeIDIdxMap[fromNode.ID()] = fromIdx - } - toIdx, toBuilderExists := dp.nodeIDIdxMap[e.To().ID()] - if !toBuilderExists { - dp.annMap[toTableExpr] = toAnnotation - toAnnotation.SetDynamic() - var toErr error - toIdx, toErr = dp.orchestrate( - -1, toAnnotation, toInsPsc, arrivingDestinationNodeStream, departingDestinationNodeStream) - if toErr != nil { - return toErr - } - dp.nodeIDIdxMap[e.To().ID()] = toIdx - } - if !dp.dataflowEdgeExists(fromIdx, toIdx) { - dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) - } - } - for _, n := range orderedNodes { - // another pass for AOT dataflows; to wit, on clauses - if _, ok := idsVisited[n.ID()]; ok { - continue - } - for _, e := range edges { - toIdx, toFound := dp.nodeIDIdxMap[e.To().ID()] - if !toFound { - return fmt.Errorf("unknown to node index") - } - fromIdx, fromFound := dp.nodeIDIdxMap[e.From().ID()] - if !fromFound { - return fmt.Errorf("unknown from node index") - } - dp.dataflowToEdges[toIdx] = append(dp.dataflowToEdges[toIdx], fromIdx) - } - } - default: - return fmt.Errorf("cannot support dependency unit of type = '%T'", unit) - } - } - maxWeaklyConnectedComponents := dp.handlerCtx.GetRuntimeContext().DataflowComponentsMax - if weaklyConnectedComponentCount > maxWeaklyConnectedComponents { - return fmt.Errorf( - "data flow: there are too many weakly connected components; found = %d, max = 1", - weaklyConnectedComponentCount) - } - rewrittenWhereStr := astvisit.GenerateModifiedWhereClause( - dp.annotatedAST, - dp.rewrittenWhere, - dp.handlerCtx.GetSQLSystem(), - dp.handlerCtx.GetASTFormatter(), - dp.handlerCtx.GetNamespaceCollection()) - rewrittenWhereStr, err = dp.handlerCtx.GetSQLSystem().SanitizeWhereQueryString(rewrittenWhereStr) - if err != nil { - return err - } - logging.GetLogger().Debugf("rewrittenWhereStr = '%s'", rewrittenWhereStr) - drmCfg, err := drm.GenerateDRMConfig( - dp.handlerCtx.GetSQLSystem(), - dp.handlerCtx.GetPersistenceSystem(), - dp.handlerCtx.GetTypingConfig(), - dp.handlerCtx.GetNamespaceCollection(), - dp.handlerCtx.GetControlAttributes()) - if err != nil { - return err - } - v := astvisit.NewQueryRewriteAstVisitor( - dp.annotatedAST, - dp.handlerCtx, - dp.tblz, - dp.tableSlice, - dp.annMap, - dp.discoGenIDs, - dp.colRefs, - drmCfg, - dp.primaryTcc, - dp.secondaryTccs, - rewrittenWhereStr, - drmCfg.GetNamespaceCollection(), - ).WithFormatter(drmCfg.GetSQLSystem().GetASTFormatter()).WithPrepStmtOffset(dp.prepStmtOffset) - err = v.Visit(dp.sqlStatement) - if err != nil { - return err - } - selCtx, err := v.GenerateSelectDML() - if err != nil { - return err - } - selBld := primitivebuilder.NewSingleSelect( - dp.primitiveComposer.GetGraphHolder(), - dp.handlerCtx, - selCtx, - dp.tableSlice, - nil, - streaming.NewNopMapStream(), - ) - selIndirect, indirectErr := astindirect.NewParserSelectIndirect(dp.sqlStatement, selCtx) - if indirectErr != nil { - return indirectErr - } - dp.annotatedAST.SetSelectIndirect(dp.sqlStatement, selIndirect) - if dp.isElideRead { - selBld = primitivebuilder.NewNopBuilder( - dp.primitiveComposer.GetGraphHolder(), - dp.primitiveComposer.GetTxnCtrlCtrs(), - dp.handlerCtx, - dp.handlerCtx.GetSQLEngine(), - []string{}, - ) - } - // TODO: make this finer grained STAT - dp.bldr = primitivebuilder.NewDependentMultipleAcquireAndSelect( - dp.primitiveComposer.GetGraphHolder(), - dp.execSlice, - selBld, - dp.dataflowToEdges, - dp.handlerCtx.GetSQLSystem(), - ) - dp.selCtx = selCtx - return nil -} - -func (dp *standardDependencyPlanner) processOrphan( - sqlNode sqlparser.SQLNode, - annotationCtx taxonomy.AnnotationCtx, - vertex dataflow.Vertex, -) (drm.PreparedStatementCtx, internaldto.TxnControlCounters, error) { - anTab, tcc, err := dp.processAcquire(sqlNode, annotationCtx, vertex) - if err != nil { - return nil, nil, err - } - - tableMetadata := annotationCtx.GetTableMeta() - - _, isSQLDataSource := annotationCtx.GetTableMeta().GetSQLDataSource() - var opStore formulation.StandardOperationStore - if !isSQLDataSource { - opStore, err = annotationCtx.GetTableMeta().GetMethod() - if err != nil { - return nil, nil, err - } - } else { - // Persist SQL mirror table here prior to generating insert DML - drmCfg := dp.handlerCtx.GetDrmConfig() - ddl, ddlErr := drmCfg.GenerateDDL(anTab, nil, nil, nil, opStore, annotationCtx.IsAwait(), 0, false, false) - if ddlErr != nil { - return nil, nil, ddlErr - } - err = dp.handlerCtx.GetSQLEngine().ExecInTxn(ddl) - if err != nil { - return nil, nil, err - } - - insPsc, insPscErr := dp.primitiveComposer.GetDRMConfig().GenerateInsertDML( - anTab, - nil, - nil, - nil, - opStore, - tcc, - false, - annotationCtx.IsAwait(), - ) - return insPsc, tcc, insPscErr - } - - anySdkProv, anySdkPrvErr := tableMetadata.GetProviderObject() - if anySdkPrvErr != nil { - return nil, nil, anySdkPrvErr - } - svc, svcErr := tableMetadata.GetService() - if svcErr != nil { - return nil, nil, svcErr - } - resource, resourceErr := tableMetadata.GetResource() - if resourceErr != nil { - return nil, nil, resourceErr - } - insPsc, err := dp.primitiveComposer.GetDRMConfig().GenerateInsertDML( - anTab, - anySdkProv, - svc, - resource, - opStore, - tcc, - false, - annotationCtx.IsAwait(), - ) - return insPsc, tcc, err -} - -func (dp *standardDependencyPlanner) orchestrate( - equivalencyGroupID int64, - annotationCtx taxonomy.AnnotationCtx, - insPsc drm.PreparedStatementCtx, - inStream streaming.MapStream, - outStream streaming.MapStream, -) (int, error) { - rc, err := tableinsertioncontainer.NewTableInsertionContainer( - annotationCtx.GetTableMeta(), - dp.handlerCtx.GetSQLEngine(), - dp.handlerCtx.GetTxnCounterMgr(), - ) - if equivalencyGroupID > 0 { - tcc, ok := dp.equivalencyGroupTCCs[equivalencyGroupID] - if ok { - tn, _ := rc.GetTableTxnCounters() - setErr := rc.SetTableTxnCounters(tn, tcc) - if setErr != nil { - return -1, setErr - } - } - } - if err != nil { - return -1, err - } - sqlDataSource, isSQLDataSource := annotationCtx.GetTableMeta().GetSQLDataSource() - var builder primitivebuilder.Builder - if isSQLDataSource { - // TODO: generate query properly with ordered columns - // starColNames := range insPsc.GetNonControlColumns() - var colNames []string - colz := insPsc.GetNonControlColumns() - for _, col := range colz { - // - colNames = append(colNames, fmt.Sprintf(`"%s"`, col.GetIdentifier())) - } - projectionStr := strings.Join(colNames, ", ") - _, tErr := dp.handlerCtx.GetDrmConfig().GetCurrentTable(annotationCtx.GetHIDs()) - if tErr != nil { - return -1, tErr - } - tableName := annotationCtx.GetHIDs().GetSQLDataSourceTableName() - // targetTableName := annotationCtx.GetHIDs().GetStackQLTableName() - query := fmt.Sprintf(`SELECT %s FROM %s`, projectionStr, tableName) - if sqlDataSource.GetDBName() == constants.SQLDbNameSnowflake { - query = strings.ReplaceAll(query, `"`, ``) - } - builder = primitivebuilder.NewSQLDataSourceSingleSelectAcquire( - dp.primitiveComposer.GetGraphHolder(), - dp.handlerCtx, - rc, - query, - nil, - insPsc, - nil, - // this is the connector - outStream, - ) - } else { - bldrInput := builder_input.NewBuilderInput( - dp.primitiveComposer.GetGraphHolder(), - dp.handlerCtx, - annotationCtx.GetTableMeta(), - ) - bldrInput.SetIsAwait(false) // returning hardcoded to false for now - setAcquirePushdownPlan(bldrInput, annotationCtx, dp.sqlStatement) - builder = primitivebuilder.NewSingleSelectAcquire( - dp.primitiveComposer.GetGraphHolder(), - dp.handlerCtx, - rc, - insPsc, - nil, - // this is the connector - outStream, - bldrInput, - ) - } - dp.execSlice = append(dp.execSlice, builder) - idx := len(dp.execSlice) - 1 - dp.tableSlice = append(dp.tableSlice, rc) - err = annotationCtx.Prepare(dp.handlerCtx, inStream) - return idx, err -} - -// setAcquirePushdownPlan records the analysis-phase query-option push-down decision for a -// SELECT acquire. The SELECT's WHERE / ORDER BY / LIMIT / OFFSET / projection / COUNT become -// a neutral PushdownIntent stored on the table metadata; the HTTP request-prepare phase hands -// it to the preparator, which (inside any-sdk) translates it to the method's dialect and sets -// the request query. A pushable LIMIT is also carried on the plan as the GraphQL page-size -// bound. Both are carried out downstream - the executor stays protocol-agnostic - and the -// client-side WHERE / projection / LIMIT are unchanged, so push-down is only ever an -// optimisation. -func setAcquirePushdownPlan( - bldrInput builder_input.BuilderInput, - annotationCtx taxonomy.AnnotationCtx, - stmt *sqlparser.Select, -) { - tableMeta := annotationCtx.GetTableMeta() - method, _ := tableMeta.GetMethod() - if intent, ok := pushdown.ComputeIntent(stmt, method); ok { - tableMeta.SetPushdownIntent(intent) - } - if limit, ok := pushdown.SelectLimit(stmt); ok { - bldrInput.SetPushdownLimit(limit) - } -} - -//nolint:gocognit,funlen // live with it -func (dp *standardDependencyPlanner) processAcquire( - sqlNode sqlparser.SQLNode, - annotationCtx taxonomy.AnnotationCtx, - vertex dataflow.Vertex, -) (util.AnnotatedTabulation, internaldto.TxnControlCounters, error) { - inputTableName, err := annotationCtx.GetInputTableName() - inputProviderString := annotationCtx.GetHIDs().GetProviderStr() - sqlDataSource, isSQLDataSource := dp.handlerCtx.GetSQLDataSource(inputProviderString) - if isSQLDataSource { - if dp.tcc == nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), - nil, - fmt.Errorf("nil counters disallowed in dependency planner") - } - if !dp.tccSetAheadOfTime { - dp.tcc = dp.tcc.CloneAndIncrementInsertID() - } - dp.secondaryTccs = append(dp.secondaryTccs, dp.tcc) - anTab := util.NewAnnotatedTabulation( - nil, annotationCtx.GetHIDs(), - inputTableName, - annotationCtx.GetTableMeta().GetAlias()) - // TODO: repeat this for mutations - anTab.SetSQLDataSource(sqlDataSource) - return anTab, dp.tcc, nil - } - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - prov, err := annotationCtx.GetTableMeta().GetProviderObject() - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - svc, err := annotationCtx.GetTableMeta().GetService() - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - resource, rscErr := annotationCtx.GetTableMeta().GetResource() - if rscErr != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, rscErr - } - m, err := annotationCtx.GetTableMeta().GetMethod() - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - selectItemsKey := annotationCtx.GetTableMeta().GetSelectItemsKey() - if selectItemsKey == "" { - selectItemsKey = m.GetSelectItemsKey() - } - var defaultColName string - if selectItemsKey != "" { - defaultColName = util.TrimSelectItemsKey(selectItemsKey) - } - tab := annotationCtx.GetSchema().Tabulate(false, defaultColName) - _, mediaType, err := m.GetResponseBodySchemaAndMediaType() - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - switch mediaType { - case media.MediaTypeTextXML, media.MediaTypeXML: - tab = tab.RenameColumnsToXml() - } - // TODO: support defaulting columns to where parameters - anTab := util.NewAnnotatedTabulation( - tab, - annotationCtx.GetHIDs(), - inputTableName, - annotationCtx.GetTableMeta().GetAlias()).WithParameters(annotationCtx.GetParameters()) - - discoGenID, err := docparser.OpenapiStackQLTabulationsPersistor( - prov, - svc, - resource, - m, - annotationCtx.IsAwait(), - []util.AnnotatedTabulation{anTab}, - dp.primitiveComposer.GetSQLEngine(), - prov.GetName(), - dp.handlerCtx.GetNamespaceCollection(), - dp.handlerCtx.GetControlAttributes(), - dp.handlerCtx.GetSQLSystem(), - dp.handlerCtx.GetPersistenceSystem(), - dp.handlerCtx.GetTypingConfig(), - ) - if err != nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err - } - dp.discoGenIDs[sqlNode] = discoGenID - if dp.tcc == nil { - return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, fmt.Errorf("nil counters disallowed in dependency planner") - } - if !dp.tccSetAheadOfTime { - if vertex.GetEquivalencyGroup() > 0 { - tcc, ok := dp.equivalencyGroupTCCs[vertex.GetEquivalencyGroup()] - if ok { - dp.tcc = tcc.Clone() - } else { - dp.tcc = dp.tcc.CloneAndIncrementInsertID() - dp.secondaryTccs = append(dp.secondaryTccs, dp.tcc) - dp.equivalencyGroupTCCs[vertex.GetEquivalencyGroup()] = dp.tcc - } - return anTab, dp.tcc, nil - } - dp.tcc = dp.tcc.CloneAndIncrementInsertID() - } - dp.secondaryTccs = append(dp.secondaryTccs, dp.tcc) - return anTab, dp.tcc, nil -} - -func (dp *standardDependencyPlanner) isVectorParam(param interface{}) bool { - paramMeta, isParamMeta := param.(parserutil.ParameterMetadata) - if isParamMeta { - val := paramMeta.GetVal() - _, valIsSQLVal := val.(sqlparser.ValTuple) - if valIsSQLVal { - return true - } - } - return false -} - -//nolint:nestif // live with it -func (dp *standardDependencyPlanner) getSQLStreamFromEdge( - e dataflow.Edge, - toAc taxonomy.AnnotationCtx, - tcc internaldto.TxnControlCounters, -) (streaming.MapStream, error) { - columnDescriptors, err := e.GetColumnDescriptors() - if err != nil { - return nil, err - } - selectCtx, err := dp.generateSelectDML(e.GetSource().GetAnnotation(), columnDescriptors, tcc, nil) - if err != nil { - return nil, err - } - ann := e.GetSource().GetAnnotation() - meta := ann.GetTableMeta() - insertContainer, err := tableinsertioncontainer.NewTableInsertionContainer(meta, - dp.handlerCtx.GetSQLEngine(), - dp.handlerCtx.GetTxnCounterMgr()) - if err != nil { - return nil, err - } - toParams := toAc.GetParameters() - - transformedToStaticParams, paramTransformErr := util.TransformSQLRawParameters(toParams, false) - if paramTransformErr != nil { - return nil, paramTransformErr - } - for k, v := range transformedToStaticParams { - hasSourceAttr := e.HasSourceAttribute(k) - if !hasSourceAttr { - vIsUnpopulated := k == v - if vIsUnpopulated { - delete(transformedToStaticParams, k) - } - } - } - return sqlstream.NewSimpleSQLMapStream( - selectCtx, - insertContainer, - dp.handlerCtx.GetDrmConfig(), - dp.handlerCtx.GetSQLEngine(), - transformedToStaticParams, - ), nil -} - -//nolint:gocognit // live with it -func (dp *standardDependencyPlanner) getStreamFromEdge( - e dataflow.Edge, - toAc taxonomy.AnnotationCtx, - tcc internaldto.TxnControlCounters, -) (streaming.MapStream, error) { - if e.IsSQL() { - return dp.getSQLStreamFromEdge(e, toAc, tcc) - } - projection, err := e.GetProjection() - if err != nil { - return nil, err - } - method, err := toAc.GetTableMeta().GetMethod() - if err != nil { - return nil, err - } - requestParameters := method.GetParametersIncludingNativeCasing() - sourceFilters, err := dp.sourceOnlyFilters(e.GetSource().GetAnnotation()) - if err != nil { - return nil, err - } - sqlProjection := destinationRequestProjection(projection, requestParameters) - projection = sourceResponseProjection(projection, e.GetSource().GetAnnotation().GetSchema().Tabulate(false, "")) - projection = destinationRequestProjection(projection, requestParameters) - incomingCols := make(map[string]struct{}) - for _, v := range projection { - incomingCols[v] = struct{}{} - } - for alias, parameter := range requestParameters { - if _, dynamic := incomingCols[parameter.GetName()]; dynamic { - incomingCols[alias] = struct{}{} - } - } - params := toAc.GetParameters() - staticParams := make(map[string]interface{}) - for k, v := range params { - isVector := dp.isVectorParam(v) - if _, ok := incomingCols[k]; !ok && !isVector { - staticParams[k] = v - incomingCols[k] = struct{}{} - } - } - if len(staticParams) > 0 { - staticParams, err = util.TransformSQLRawParameters(staticParams, false) - if err != nil { - return nil, err - } - } - if len(sourceFilters) > 0 { - return dp.filteredSourceStream(e.GetSource().GetAnnotation(), sqlProjection, staticParams, sourceFilters, tcc) - } - return streaming.NewSimpleProjectionMapStream(projection, staticParams), nil -} - -// filteredSourceStream reads the dependent table's request parameters from -// the source table's stored rows, restricted by the source only filters, so -// that no request is made for a source row that the WHERE clause discards. -func (dp *standardDependencyPlanner) filteredSourceStream( - sourceAnnotation taxonomy.AnnotationCtx, - projection map[string]string, - staticParams map[string]interface{}, - filters []string, - tcc internaldto.TxnControlCounters, -) (streaming.MapStream, error) { - alias := sourceAnnotation.GetTableMeta().GetAlias() - var columnDescriptors []formulation.ColumnDescriptor - for source, destination := range projection { - sourceColumn := &sqlparser.ColName{ - Name: sqlparser.NewColIdent(source), - Qualifier: sqlparser.TableName{Name: sqlparser.NewTableIdent(alias)}, - } - decoratedColumn := fmt.Sprintf(`%s AS "%s"`, sqlparser.ColDelimitedString(sourceColumn), destination) - columnDescriptors = append( - columnDescriptors, - formulation.NewColumnDescriptor(destination, "", "", decoratedColumn, nil, nil, nil), - ) - } - selectCtx, err := dp.generateSelectDML(sourceAnnotation, columnDescriptors, tcc, filters) - if err != nil { - return nil, err - } - insertContainer, err := tableinsertioncontainer.NewTableInsertionContainer( - sourceAnnotation.GetTableMeta(), - dp.handlerCtx.GetSQLEngine(), - dp.handlerCtx.GetTxnCounterMgr()) - if err != nil { - return nil, err - } - return sqlstream.NewSimpleSQLMapStream( - selectCtx, - insertContainer, - dp.handlerCtx.GetDrmConfig(), - dp.handlerCtx.GetSQLEngine(), - staticParams, - ), nil -} - -// sourceOnlyFilters returns the top level WHERE conjuncts that refer only to -// response columns of the source table, rendered for its stored rows. -// It returns none when the source table may be null extended by an outer -// join, since a WHERE clause may then retain rows the source table does not -// match. Conjuncts on request parameters are already enforced by the -// source request itself, and unqualified columns are not attributed. -func (dp *standardDependencyPlanner) sourceOnlyFilters(sourceAnnotation taxonomy.AnnotationCtx) ([]string, error) { - where := dp.sqlStatement.Where - if where == nil || where.Type != sqlparser.WhereStr || isAnnotationIndirection(sourceAnnotation) { - return nil, nil - } - alias := sourceAnnotation.GetTableMeta().GetAlias() - if alias == "" || isNullExtended(dp.sqlStatement.From, alias, false) { - return nil, nil - } - method, err := sourceAnnotation.GetTableMeta().GetMethod() - if err != nil { - return nil, err - } - requestParameters := make(map[string]struct{}) - for name, parameter := range method.GetParametersIncludingNativeCasing() { - requestParameters[name] = struct{}{} - requestParameters[parameter.GetName()] = struct{}{} - } - for name := range sourceAnnotation.GetParameters() { - requestParameters[name] = struct{}{} - } - var filters []string - for _, conjunct := range sqlparser.SplitAndExpression(nil, where.Expr) { - if isSourceOnlyResponseExpr(conjunct, alias, requestParameters) { - filters = append(filters, astformat.String(conjunct, dp.handlerCtx.GetASTFormatter())) - } - } - return filters, nil -} - -func isSourceOnlyResponseExpr(expr sqlparser.Expr, alias string, requestParameters map[string]struct{}) bool { - columnCount := 0 - eligible := true - //nolint:errcheck // the visitor never returns an error - sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { - switch node := node.(type) { - case *sqlparser.ColName: - _, isRequestParameter := requestParameters[node.Name.GetRawVal()] - if node.Qualifier.Name.GetRawVal() != alias || !node.Qualifier.Qualifier.IsEmpty() || isRequestParameter { - eligible = false - } - columnCount++ - case *sqlparser.Subquery, *sqlparser.ExistsExpr, *sqlparser.ValuesFuncExpr, *sqlparser.Default: - eligible = false - } - return eligible, nil - }, expr) - return eligible && columnCount > 0 -} - -// isNullExtended reports whether the table aliased by alias sits on the -// optional side of an outer join within the table expressions. -func isNullExtended(tableExprs sqlparser.TableExprs, alias string, isOptionalSide bool) bool { - for _, tableExpr := range tableExprs { - switch tableExpr := tableExpr.(type) { - case *sqlparser.AliasedTableExpr: - if tableExpr.As.GetRawVal() == alias { - return isOptionalSide - } - case *sqlparser.ParenTableExpr: - if isNullExtended(tableExpr.Exprs, alias, isOptionalSide) { - return true - } - case *sqlparser.JoinTableExpr: - leftOptional, rightOptional := isOptionalSide, isOptionalSide - switch tableExpr.Join { - case sqlparser.LeftJoinStr, sqlparser.LeftOuterJoinStr, sqlparser.NaturalLeftJoinStr: - rightOptional = true - case sqlparser.RightJoinStr, sqlparser.RightOuterJoinStr, sqlparser.NaturalRightJoinStr: - leftOptional = true - } - if isNullExtended(sqlparser.TableExprs{tableExpr.LeftExpr}, alias, leftOptional) || - isNullExtended(sqlparser.TableExprs{tableExpr.RightExpr}, alias, rightOptional) { - return true - } - } - } - return false -} - -func sourceResponseProjection(projection map[string]string, tab formulation.Tabulation) map[string]string { - wireNames := make(map[string]string) - for _, column := range tab.GetColumns() { - wireNames[column.GetName()] = column.GetWireName() - } - translated := make(map[string]string, len(projection)) - for source, destination := range projection { - if wireName := wireNames[source]; wireName != "" { - source = wireName - } - translated[source] = destination - } - return translated -} - -func destinationRequestProjection( - projection map[string]string, - parameters map[string]formulation.Addressable, -) map[string]string { - translated := make(map[string]string, len(projection)) - for source, destination := range projection { - if parameter, ok := parameters[destination]; ok { - destination = parameter.GetName() - } - translated[source] = destination - } - return translated -} - -// This is very naive but safe because there is only ever one source table. -func (dp *standardDependencyPlanner) harvestFilter(sourceAnnotation taxonomy.AnnotationCtx) (string, bool) { - var valz []string - for k, p := range sourceAnnotation.GetParameters() { - //nolint:gocritic // fine with this - switch pt := p.(type) { - case *sqlparser.SQLVal: - val := string(pt.Val) - s := fmt.Sprintf(`"%s" = '%s' `, k, val) - valz = append(valz, s) - } - } - if len(valz) == 0 { - return "", false - } - rv := strings.Join(valz, " AND ") - return rv, true -} - -func (dp *standardDependencyPlanner) generateSelectDML( - ann taxonomy.AnnotationCtx, - columnDescriptors []formulation.ColumnDescriptor, - tcc internaldto.TxnControlCounters, - filters []string, -) (drm.PreparedStatementCtx, error) { - whereStr, _ := dp.harvestFilter(ann) - for _, filter := range filters { - if whereStr != "" { - whereStr += " AND " - } - whereStr += fmt.Sprintf("(%s)", filter) - } - discoID, discoIDErr := dp.handlerCtx.GetSQLEngine().GetCurrentDiscoveryGenerationID(ann.GetHIDs().GetProviderStr()) - if discoIDErr != nil { - return nil, fmt.Errorf("error generating select dml: %w", discoIDErr) - } - alias := ann.GetTableMeta().GetAlias() - tn, err := dp.handlerCtx.GetDrmConfig().GetTable(ann.GetHIDs(), discoID) - if err != nil { - return nil, err - } - tableName := fmt.Sprintf(`"%s"`, tn.GetName()) - // TODO: obtain namespace prefix for postgres - tableSchema := tn.GetNameSpace() - if tableSchema != "" { - tableName = fmt.Sprintf(`"%s"."%s"`, tableSchema, tn.GetName()) - } - if alias != "" { - tableName = fmt.Sprintf("%s AS %s", tableName, alias) - } - relationalColumns := dp.handlerCtx.GetDrmConfig().OpenapiColumnsToRelationalColumns(columnDescriptors) - rewriteInput := sqlrewrite.NewStandardSQLRewriteInput( - dp.handlerCtx.GetDrmConfig(), - relationalColumns, - tcc, - "", - whereStr, - dp.secondaryTccs, - dp.tblz, - tableName, // this is the sole from string - nil, - dp.handlerCtx.GetNamespaceCollection(), - nil, - map[string]interface{}{}, - ).WithSelectQualifier("DISTINCT") - return sqlrewrite.GenerateRewrittenSelectDML(rewriteInput) -} diff --git a/internal/stackql/dependencyplanner/dependencyplanner_test.go b/internal/stackql/dependencyplanner/dependencyplanner_test.go deleted file mode 100644 index d291f2900..000000000 --- a/internal/stackql/dependencyplanner/dependencyplanner_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package dependencyplanner //nolint:testpackage // tests internal projection mapping - -import ( - "io" - "reflect" - "testing" - - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -type projectionColumn struct { - formulation.ColumnDescriptor - name string - wireName string -} - -func (column projectionColumn) GetName() string { return column.name } -func (column projectionColumn) GetWireName() string { return column.wireName } - -type projectionTabulation struct { - formulation.Tabulation - columns []formulation.ColumnDescriptor -} - -func (tab projectionTabulation) GetColumns() []formulation.ColumnDescriptor { return tab.columns } - -type projectionParameter struct { - formulation.Addressable - name string -} - -func (parameter projectionParameter) GetName() string { return parameter.name } - -func TestDestinationRequestProjection(t *testing.T) { - projection := map[string]string{"AccessKeyId": "access_key_id", "other": "unchanged"} - parameters := map[string]formulation.Addressable{ - "access_key_id": projectionParameter{name: "AccessKeyId"}, - } - translated := destinationRequestProjection(projection, parameters) - want := map[string]string{"AccessKeyId": "AccessKeyId", "other": "unchanged"} - if !reflect.DeepEqual(translated, want) { - t.Fatalf("projection = %v, want %v", translated, want) - } - if projection["AccessKeyId"] != "access_key_id" { - t.Fatal("source projection was mutated") - } -} - -func TestSourceResponseProjection(t *testing.T) { - projection := map[string]string{ - "access_key_id": "access_key_id", - "unchanged": "destination", - "unknown": "unknown_destination", - "empty_wire": "empty_destination", - } - tab := projectionTabulation{columns: []formulation.ColumnDescriptor{ - projectionColumn{name: "access_key_id", wireName: "AccessKeyId"}, - projectionColumn{name: "unchanged", wireName: "unchanged"}, - projectionColumn{name: "empty_wire"}, - }} - translated := sourceResponseProjection(projection, tab) - want := map[string]string{ - "AccessKeyId": "access_key_id", "unchanged": "destination", - "unknown": "unknown_destination", "empty_wire": "empty_destination", - } - if !reflect.DeepEqual(translated, want) { - t.Fatalf("projection = %v, want %v", translated, want) - } - if projection["access_key_id"] != "access_key_id" { - t.Fatal("source projection was mutated") - } - stream := streaming.NewSimpleProjectionMapStream(translated, nil) - if err := stream.Write([]map[string]interface{}{{ - "AccessKeyId": "test-key", "unchanged": "value", "unknown": "other", "empty_wire": "empty", - }}); err != nil { - t.Fatal(err) - } - rows, err := stream.Read() - if err != io.EOF { - t.Fatalf("stream read: %v", err) - } - if len(rows) != 1 || rows[0]["access_key_id"] != "test-key" { - t.Fatalf("projected rows = %v", rows) - } -} - -func parseSelect(t *testing.T, query string) *sqlparser.Select { - t.Helper() - stmt, err := sqlparser.Parse(query) - if err != nil { - t.Fatalf("parse error: %v", err) - } - selectStmt, ok := stmt.(*sqlparser.Select) - if !ok { - t.Fatalf("not a select: %T", stmt) - } - return selectStmt -} - -func TestIsSourceOnlyResponseExpr(t *testing.T) { - requestParameters := map[string]struct{}{"org": {}} - for where, expected := range map[string]bool{ - "r.archived = 0": true, - "r.archived = 0 and r.size > 1": true, - "r.org = 'stackql'": false, - "archived = 0": false, - "b.type = 'x'": false, - "r.name = b.repo": false, - "1 = 1": false, - "r.name in (select name from t)": false, - } { - stmt := parseSelect(t, "select 1 from t where "+where) - if got := isSourceOnlyResponseExpr(stmt.Where.Expr, "r", requestParameters); got != expected { - t.Errorf("isSourceOnlyResponseExpr(%q) = %t, want %t", where, got, expected) - } - } -} - -func TestIsNullExtended(t *testing.T) { - for from, expected := range map[string]bool{ - "a r inner join b on r.x = b.x": false, - "a r left outer join b on r.x = b.x": false, - "b left outer join a r on r.x = b.x": true, - "a r right join b on r.x = b.x": true, - "c inner join (b left join a r on r.x = b.x) on c.y = b.y": true, - } { - stmt := parseSelect(t, "select 1 from "+from) - if got := isNullExtended(stmt.From, "r", false); got != expected { - t.Errorf("isNullExtended(%q) = %t, want %t", from, got, expected) - } - } -} diff --git a/internal/stackql/dependencyplanner/stream_collection.go b/internal/stackql/dependencyplanner/stream_collection.go deleted file mode 100644 index 7deb14aa9..000000000 --- a/internal/stackql/dependencyplanner/stream_collection.go +++ /dev/null @@ -1,54 +0,0 @@ -package dependencyplanner - -import ( - "github.com/stackql/any-sdk/pkg/streaming" -) - -type StreamDependecyCollection interface { - // Add adds a stream to the collection - // departingID is the ID of the stream that is departing - // arrivingID is the ID of the stream that is arriving - // stream is the stream that is being added - Add(departingID int64, arrivingID int64, stream streaming.MapStream) - GetArriving(int64) streaming.MapStreamCollection - GetDeparting(int64) streaming.MapStreamCollection -} - -type streamDependencyCollection struct { - departingStreams map[int64][]streaming.MapStream - arrivingStreams map[int64][]streaming.MapStream -} - -func NewStreamDependecyCollection() StreamDependecyCollection { - return &streamDependencyCollection{ - departingStreams: make(map[int64][]streaming.MapStream), - arrivingStreams: make(map[int64][]streaming.MapStream), - } -} - -func (sdc *streamDependencyCollection) Add(departingID int64, arrivingID int64, stream streaming.MapStream) { - sdc.departingStreams[departingID] = append(sdc.departingStreams[departingID], stream) - sdc.arrivingStreams[arrivingID] = append(sdc.arrivingStreams[arrivingID], stream) -} - -func (sdc *streamDependencyCollection) GetArriving(id int64) streaming.MapStreamCollection { - rv := streaming.NewStandardMapStreamCollection() - for _, stream := range sdc.arrivingStreams[id] { - rv.Push(stream) - } - if rv.Len() == 0 { - rv.Push(streaming.NewStandardMapStream()) - } - return rv -} - -func (sdc *streamDependencyCollection) GetDeparting(id int64) streaming.MapStreamCollection { - rv := streaming.NewStandardMapStreamCollection() - for _, stream := range sdc.departingStreams[id] { - rv.Push(stream) - } - if rv.Len() == 0 { - rv.Push(streaming.NewNopMapStream()) - } - return rv -} diff --git a/internal/stackql/docparser/doc_parser.go b/internal/stackql/docparser/doc_parser.go deleted file mode 100644 index 62c3f75d5..000000000 --- a/internal/stackql/docparser/doc_parser.go +++ /dev/null @@ -1,94 +0,0 @@ -package docparser - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/sql_system" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/typing" - "github.com/stackql/stackql/internal/stackql/util" - - "strings" -) - -const ( - SchemaDelimiter string = "." - googleServiceKeyDelimiter string = ":" - stackqlServiceKeyDelimiter string = "__" -) - -func TranslateServiceKeyGenericProviderToIql(serviceKey string) string { - //nolint:gocritic // TODO: review - return strings.Replace(serviceKey, googleServiceKeyDelimiter, stackqlServiceKeyDelimiter, -1) -} - -func TranslateServiceKeyIqlToGenericProvider(serviceKey string) string { - //nolint:gocritic // TODO: review - return strings.Replace(serviceKey, stackqlServiceKeyDelimiter, googleServiceKeyDelimiter, -1) -} - -func OpenapiStackQLTabulationsPersistor( - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - m formulation.StandardOperationStore, - isAwait bool, - tabluationsAnnotated []util.AnnotatedTabulation, - dbEngine sqlengine.SQLEngine, - prefix string, - namespaceCollection tablenamespace.Collection, - controlAttributes sqlcontrol.ControlAttributes, - sqlSystem sql_system.SQLSystem, - persistenceSystem formulation.PersistenceSystem, - typCfg typing.Config, -) (int, error) { - drmCfg, err := drm.GenerateDRMConfig(sqlSystem, persistenceSystem, - typCfg, namespaceCollection, controlAttributes) - if err != nil { - return 0, err - } - discoveryGenerationID, err := dbEngine.GetCurrentDiscoveryGenerationID(prefix) - if err != nil { - discoveryGenerationID, err = dbEngine.GetNextDiscoveryGenerationID(prefix) - if err != nil { - return discoveryGenerationID, err - } - } - db, err := dbEngine.GetDB() - if err != nil { - return discoveryGenerationID, err - } - txn, err := db.Begin() - if err != nil { - return discoveryGenerationID, err - } - for _, tblt := range tabluationsAnnotated { - ddl, ddlErr := drmCfg.GenerateDDL(tblt, prov, svc, resource, m, isAwait, discoveryGenerationID, false, true) - if ddlErr != nil { - displayErr := fmt.Errorf("error generating DDL: %w", ddlErr) - logging.GetLogger().Infoln(displayErr.Error()) - txn.Rollback() //nolint:errcheck // TODO: investigate - return discoveryGenerationID, displayErr - } - for _, q := range ddl { - logging.GetLogger().Infof("DDL about to run: '''%q'''", q) - _, err = txn.Exec(q) - if err != nil { - displayErr := fmt.Errorf("aborting DDL run for query '''%s''' with error: %w", q, err) - logging.GetLogger().Infof("aborting DDL run for query '''%s''' with error: %s", q, err.Error()) - txn.Rollback() //nolint:errcheck // TODO: investigate - return discoveryGenerationID, displayErr - } - } - } - err = txn.Commit() - if err != nil { - return discoveryGenerationID, err - } - return discoveryGenerationID, nil -} diff --git a/internal/stackql/driver/aggregation_compute_disks_integration_test.go b/internal/stackql/driver/aggregation_compute_disks_integration_test.go index 4871d27b1..ef23d70b0 100644 --- a/internal/stackql/driver/aggregation_compute_disks_integration_test.go +++ b/internal/stackql/driver/aggregation_compute_disks_integration_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/logging" . "github.com/stackql/stackql/internal/stackql/driver" "github.com/stackql/stackql/internal/stackql/entryutil" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/querysubmit" "github.com/stackql/stackql/internal/stackql/responsehandler" "github.com/stackql/stackql/internal/test/stackqltestutil" diff --git a/internal/stackql/driver/dependent_simple_integration_test.go b/internal/stackql/driver/dependent_simple_integration_test.go index 3a44d081d..000385aa6 100644 --- a/internal/stackql/driver/dependent_simple_integration_test.go +++ b/internal/stackql/driver/dependent_simple_integration_test.go @@ -52,7 +52,8 @@ func TestSimpleInsertDependentGoogleComputeDiskAsync(t *testing.T) { } stackqltestutil.SetupDependentInsertGoogleComputeDisks(t) - stackqltestutil.RunCaptureTestAgainstFiles(t, testSubject, []string{testobjects.ExpectedComputeDisksDependentInsertAsyncFile}) + stackqltestutil.RunCaptureTestAgainstFiles(t, testSubject, []string{testobjects.ExpectedComputeDisksDependentInsertAsyncFile, + "test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk-parallel.txt"}) } //nolint:govet,lll // legacy test @@ -89,8 +90,9 @@ func TestSimpleInsertDependentGoogleComputeDiskAsyncReversed(t *testing.T) { dr.ProcessQuery(handlerCtx.GetRawQuery()) } - stackqltestutil.SetupDependentInsertGoogleComputeDisks(t) - stackqltestutil.RunCaptureTestAgainstFiles(t, testSubject, []string{testobjects.ExpectedComputeDisksDependentInsertAsyncFile}) + stackqltestutil.SetupDependentInsertGoogleComputeDisksReversed(t) + stackqltestutil.RunCaptureTestAgainstFiles(t, testSubject, []string{testobjects.ExpectedComputeDisksDependentInsertAsyncFile, + "test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk-parallel.txt"}) } //nolint:govet,lll // legacy test diff --git a/internal/stackql/driver/driver.go b/internal/stackql/driver/driver.go index 8d5fa2d04..dacae46f6 100644 --- a/internal/stackql/driver/driver.go +++ b/internal/stackql/driver/driver.go @@ -7,15 +7,15 @@ import ( "strings" "time" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/acid/tsm_physio" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/paramdecoder" "github.com/stackql/stackql/internal/stackql/queryshape" "github.com/stackql/stackql/internal/stackql/responsehandler" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql/pkg/txncounter" diff --git a/internal/stackql/driver/driver_defaulted_integration_test.go b/internal/stackql/driver/driver_defaulted_integration_test.go index 6348d6039..396b6a0a1 100644 --- a/internal/stackql/driver/driver_defaulted_integration_test.go +++ b/internal/stackql/driver/driver_defaulted_integration_test.go @@ -1,19 +1,18 @@ package driver_test import ( - "context" + "encoding/json" "fmt" "io" - "net" "net/http" "net/http/httptest" "runtime" "testing" - "github.com/stackql/any-sdk/pkg/dto" . "github.com/stackql/stackql/internal/stackql/driver" + "github.com/stackql/stackql/internal/stackql/intrinsic" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/util" - "github.com/stretchr/testify/assert" "github.com/stackql/stackql/internal/stackql/handler" @@ -60,7 +59,7 @@ func getRuntimeCtx(providerStr string, outputFmtStr string, testName string) (*d } //nolint:lll // legacy test -func TestDefaultedHttpClientIntegration(t *testing.T) { +func TestOmniEndpointOverrideIntegration(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("skipping test on Windows") } @@ -77,29 +76,22 @@ func TestDefaultedHttpClientIntegration(t *testing.T) { t.Fatalf("Test failed: %v", err) } - expectedHost := "my-test-bucket.s3-ap-southeast-2.amazonaws.com" - reponseCounter := 0 tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() io.Copy(io.Discard, r.Body) - assert.Equal(t, r.Host, expectedHost, "expected host does not match actual host") w.WriteHeader(http.StatusOK) reponseCounter++ })) t.Cleanup(tlsServer.Close) - baseTransport := tlsServer.Client().Transport.(*http.Transport) - dummyClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: baseTransport.TLSClientConfig, - DisableKeepAlives: true, - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return net.Dial("tcp", tlsServer.Listener.Addr().String()) - }, - }, + preview, err := json.Marshal(map[string]any{"endpoint": tlsServer.URL, "insecureSkipTLSVerify": true}) + if err != nil { + t.Fatal(err) } + intrinsic.Init(string(preview)) + t.Cleanup(func() { intrinsic.Init("") }) testingQuery := `SELECT * FROM aws.s3.bucket_acls WHERE Bucket = 'my-test-bucket' AND created_date = '2024-01-01T00:00:00Z' AND region = 'ap-southeast-2';` @@ -111,8 +103,6 @@ func TestDefaultedHttpClientIntegration(t *testing.T) { "v0.1.0", ) - handlerCtx.SetDefaultHTTPClient(dummyClient) - dr, _ := NewStackQLDriver(handlerCtx) if err != nil { diff --git a/internal/stackql/drm/drm_cfg.go b/internal/stackql/drm/drm_cfg.go index 8c5062220..e3c084415 100644 --- a/internal/stackql/drm/drm_cfg.go +++ b/internal/stackql/drm/drm_cfg.go @@ -8,21 +8,16 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/discovery" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/radix_tree_address_space" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/sqlmachinery" + "github.com/stackql/stackql/internal/stackql/streaming" "github.com/stackql/stackql/internal/stackql/tablenamespace" "github.com/stackql/stackql/internal/stackql/typing" - "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -46,17 +41,7 @@ type Config interface { ) (map[string]map[string]interface{}, map[int]map[int]interface{}) GetCurrentTable(internaldto.HeirarchyIdentifiers) (internaldto.DBTable, error) GetRelationalType(string) string - GenerateDDL( - util.AnnotatedTabulation, - formulation.Provider, - formulation.Service, - formulation.Resource, - formulation.StandardOperationStore, - bool, - int, - bool, - bool, - ) ([]string, error) + GetControlAttributes() sqlcontrol.ControlAttributes GetGolangValue(string) interface{} GetGolangSlices([]typing.ColumnMetadata) ([]interface{}, []string) @@ -64,30 +49,9 @@ type Config interface { GetParserTableName(internaldto.HeirarchyIdentifiers, int) sqlparser.TableName GetSQLSystem() sql_system.SQLSystem GetTable(internaldto.HeirarchyIdentifiers, int) (internaldto.DBTable, error) - GenerateInsertDML( - util.AnnotatedTabulation, - formulation.Provider, - formulation.Service, - formulation.Resource, - formulation.StandardOperationStore, - internaldto.TxnControlCounters, - bool, - bool, - ) (PreparedStatementCtx, error) - GenerateSelectDML( - util.AnnotatedTabulation, - formulation.Provider, - formulation.Service, - formulation.Resource, - formulation.StandardOperationStore, - internaldto.TxnControlCounters, - string, - string, - bool, - ) (PreparedStatementCtx, error) + ExecuteInsertDML(sqlengine.SQLEngine, PreparedStatementCtx, map[string]interface{}, string) (sql.Result, error) - OpenapiColumnsToRelationalColumns(cols []formulation.ColumnDescriptor) []typing.RelationalColumn - OpenapiColumnsToRelationalColumn(col formulation.ColumnDescriptor) typing.RelationalColumn + QueryDML(sqlmachinery.Querier, PreparedStatementParameterized) (*sql.Rows, error) ExecDDL( querier sqlmachinery.ExecQuerier, @@ -120,13 +84,11 @@ type Config interface { } type staticDRMConfig struct { - namespaceCollection tablenamespace.Collection - controlAttributes sqlcontrol.ControlAttributes - sqlEngine sqlengine.SQLEngine - sqlSystem sql_system.SQLSystem - typCfg typing.Config - analyzerFactoryFactory discovery.StaticAnalyzerFactoryFactory - persistenceSystem formulation.PersistenceSystem + namespaceCollection tablenamespace.Collection + controlAttributes sqlcontrol.ControlAttributes + sqlEngine sqlengine.SQLEngine + sqlSystem sql_system.SQLSystem + typCfg typing.Config } func (dc *staticDRMConfig) GetSQLSystem() sql_system.SQLSystem { @@ -148,79 +110,6 @@ func (dc *staticDRMConfig) DelimitFullyQualifiedRelationName(fqtn string) string return dc.sqlSystem.DelimitFullyQualifiedRelationName(fqtn) } -func (dc *staticDRMConfig) OpenapiColumnsToRelationalColumns( - cols []formulation.ColumnDescriptor, -) []typing.RelationalColumn { - var relationalColumns []typing.RelationalColumn - for _, col := range cols { - var typeStr string - schemaExists := false - if col.GetSchema() != nil && !col.GetSchema().IsEmpty() { - typeStr = dc.GetRelationalType(col.GetSchema().GetType()) - schemaExists = true - } else { //nolint:gocritic // defer fix - if col.GetVal() != nil { - switch col.GetVal().Type { //nolint:gocritic,exhaustive // defer fix - case sqlparser.BitVal: - } - } - } - relationalColumn := typing.NewRelationalColumn( - col.GetName(), - typeStr, - ).WithQualifier( - col.GetQualifier(), - ).WithAlias(col.GetAlias()).WithDecorated(col.GetDecoratedCol()).WithParserNode(col.GetNode()) - if schemaExists { - inferredOID := typing.GetOidForSchema(col.GetSchema()) - relationalColumn = relationalColumn.WithOID(inferredOID) - } - // TODO: Need a way to handle postgres differences. This is a fragile point - relationalColumns = append(relationalColumns, relationalColumn) - } - return relationalColumns -} - -func (dc *staticDRMConfig) ToExternalSQLRelationalColumn( - tabAnn util.AnnotatedTabulation, - colName string, -) (typing.RelationalColumn, error) { - return nil, fmt.Errorf("cannot find column '%s' for external SQL table '%s'", colName, tabAnn.GetInputTableName()) -} - -func (dc *staticDRMConfig) OpenapiColumnsToRelationalColumn( - col formulation.ColumnDescriptor, -) typing.RelationalColumn { - var typeStr string - schemaExists := false - //nolint:gocritic,exhaustive // defer fix - if col.GetSchema() != nil && !col.GetSchema().IsEmpty() { - typeStr = dc.GetRelationalType(col.GetSchema().GetType()) - schemaExists = true - } else { - if col.GetVal() != nil { - switch col.GetVal().Type { - case sqlparser.BitVal: - } - } - } - decoratedCol := col.GetDecoratedCol() - // if col.Alias != "" { - // decoratedCol = fmt.Sprintf(`%s AS "%s"`, decoratedCol, col.Alias) - // } - relationalColumn := typing.NewRelationalColumn( - col.GetName(), - typeStr, - ).WithQualifier(col.GetQualifier()).WithAlias(col.GetAlias()).WithDecorated(decoratedCol).WithParserNode(col.GetNode()) - if schemaExists { - inferredOID := typing.GetOidForSchema(col.GetSchema()) - relationalColumn = relationalColumn.WithOID(inferredOID) - } - // TODO: Need a way to handle postgres differences - - return relationalColumn -} - func (dc *staticDRMConfig) translateColDefTypeToRelationalType( col *sqlparser.ColumnDefinition) typing.RelationalColumn { relationalColumn := typing.NewRelationalColumn( @@ -430,354 +319,6 @@ func (dc *staticDRMConfig) getParserTableName( } } -func (dc *staticDRMConfig) inferColType(col formulation.Column) string { - schema := col.GetSchema() - return dc.inferColTypeFromSchema(schema) -} - -func (dc *staticDRMConfig) inferColTypeFromSchema(schema formulation.Schema) string { - relationalType := textStr - if schema != nil && !schema.IsEmpty() && schema.GetType() != "" { - relationalType = dc.GetRelationalType(schema.GetType()) - } - return relationalType -} - -func (dc *staticDRMConfig) genRelationalTableFromExternalSQLTable( - tabAnn util.AnnotatedTabulation, - discoveryGenerationID int, -) (relationaldto.RelationalTable, error) { - tableName, err := dc.getTableName(tabAnn.GetHeirarchyIdentifiers(), discoveryGenerationID) - if err != nil { - return nil, err - } - relationalTable := relationaldto.NewRelationalTable( - tabAnn.GetHeirarchyIdentifiers(), - discoveryGenerationID, - tableName, - tabAnn.GetInputTableName(), - ) - tableColumns, err := dc.sqlSystem.ObtainRelationalColumnsFromExternalSQLtable(tabAnn.GetHeirarchyIdentifiers()) - if err != nil { - return nil, err - } - for _, col := range tableColumns { - relationalTable.PushBackColumn(col) - } - return relationalTable, nil -} - -func (dc *staticDRMConfig) obtainAddressSpace( - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - m formulation.StandardOperationStore, - isAwait bool, -) (formulation.AddressSpace, error) { - addressSpaceFormulator := formulation.NewAddressSpaceFormulator( - radix_tree_address_space.NewAddressSpaceGrammar(), - prov, - svc, - resource, - m, - m.GetProjections(), - isAwait, - ) - addressSpaceErr := addressSpaceFormulator.Formulate() - if addressSpaceErr != nil { - return nil, addressSpaceErr - } - addressSpace := addressSpaceFormulator.GetAddressSpace() - if addressSpace == nil { - return nil, fmt.Errorf("failed to obtain address space") - } - return addressSpace, nil -} - -func (dc *staticDRMConfig) obtainRelationFromAddressSpace( - isAwait bool, - addressSpace formulation.AddressSpace, - isNilResponseAllowed bool, -) (formulation.Relation, error) { - inferredRelation, inferredRelationErr := addressSpace.ToRelation( - formulation.NewStandardAddressSpaceExpansionConfig( - isAwait, - true, // TODO: switch this off at the appropriate time - isNilResponseAllowed, - )) - if inferredRelationErr != nil { - return nil, inferredRelationErr - } - return inferredRelation, nil -} - -func (dc *staticDRMConfig) genRelationalTable( - tabAnn util.AnnotatedTabulation, - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - m formulation.StandardOperationStore, - isAwait bool, - discoveryGenerationID int, - isNilResponseAllowed bool, -) (relationaldto.RelationalTable, error) { - tableName, err := dc.getTableName(tabAnn.GetHeirarchyIdentifiers(), discoveryGenerationID) - if err != nil { - return nil, err - } - _, isSQLDataSource := tabAnn.GetSQLDataSource() - if isSQLDataSource { - return dc.genRelationalTableFromExternalSQLTable(tabAnn, discoveryGenerationID) - } - relationalTable := relationaldto.NewRelationalTable( - tabAnn.GetHeirarchyIdentifiers(), - discoveryGenerationID, - tableName, - tabAnn.GetInputTableName(), - ) - addressSpace, hasAddressSpace := m.GetAddressSpace() - if !hasAddressSpace { - addressSpace, err = dc.obtainAddressSpace(prov, svc, resource, m, isAwait) - if err != nil { - return nil, err - } - } - inferredRelation, inferredRelationErr := dc.obtainRelationFromAddressSpace(isAwait, addressSpace, isNilResponseAllowed) - if inferredRelationErr != nil { - return nil, inferredRelationErr - } - tableColumns := inferredRelation.GetColumns() - for _, col := range tableColumns { - colName := col.GetName() - colType := dc.inferColType(col) - // relationalType := dc.GetRelationalType(colType) - // TODO: add drm logic to infer / transform width as suplied by openapi doc - colWidth := col.GetWidth() - relationalColumn := typing.NewRelationalColumn(colName, colType).WithWidth(colWidth) - relationalTable.PushBackColumn(relationalColumn) - } - return relationalTable, nil -} - -func (dc *staticDRMConfig) GenerateDDL( - tabAnn util.AnnotatedTabulation, - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - m formulation.StandardOperationStore, - isAwait bool, - discoveryGenerationID int, - dropTable bool, - isNilResponseAllowed bool, -) ([]string, error) { - relationalTable, err := dc.genRelationalTable( - tabAnn, prov, svc, resource, m, isAwait, discoveryGenerationID, isNilResponseAllowed) - if err != nil { - return nil, err - } - return dc.sqlSystem.GenerateDDL(relationalTable, dropTable) -} - -//nolint:gocritic,govet,funlen,nestif,gocognit // defer fix; this orchestration remains intentionally cohesive -func (dc *staticDRMConfig) GenerateInsertDML( - tabAnnotated util.AnnotatedTabulation, - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - method formulation.StandardOperationStore, - tcc internaldto.TxnControlCounters, - isNilResponseAllowed bool, - isAsync bool, -) (PreparedStatementCtx, error) { - var columns []typing.ColumnMetadata - _, isSQLDataSource := tabAnnotated.GetSQLDataSource() - var tableName string - var discoverID int - var err error - if isSQLDataSource { - tableObj, err := dc.GetCurrentTable(tabAnnotated.GetHeirarchyIdentifiers()) - tableName = tableObj.GetName() - discoverID = tableObj.GetDiscoveryID() - if err != nil { - return nil, err - } - } else { - tableObj, err := dc.GetCurrentTable(tabAnnotated.GetHeirarchyIdentifiers()) - tableName = tableObj.GetName() - discoverID = tableObj.GetDiscoveryID() - if err != nil { - return nil, err - } - } - genIDColName := dc.controlAttributes.GetControlGenIDColumnName() - sessionIDColName := dc.controlAttributes.GetControlSsnIDColumnName() - txnIDColName := dc.controlAttributes.GetControlTxnIDColumnName() - insIDColName := dc.controlAttributes.GetControlInsIDColumnName() - insEncodedColName := dc.controlAttributes.GetControlInsertEncodedIDColumnName() - - relationalTable := relationaldto.NewRelationalTable( - tabAnnotated.GetHeirarchyIdentifiers(), - discoverID, - tableName, - tabAnnotated.GetInputTableName(), - ) - if isSQLDataSource { - tableColumns, err := dc.sqlSystem.ObtainRelationalColumnsFromExternalSQLtable(tabAnnotated.GetHeirarchyIdentifiers()) - if err != nil { - return nil, err - } - for _, col := range tableColumns { - columns = append(columns, typing.NewRelayedColDescriptor(col, col.GetType())) - relationalTable.PushBackColumn(col) - } - } else { - addressSpace, addressSpaceErr := dc.obtainAddressSpace(prov, svc, resource, method, isAsync) - if addressSpaceErr != nil { - return nil, addressSpaceErr - } - inferredRelation, relationErr := dc.obtainRelationFromAddressSpace( - isAsync, - addressSpace, - isNilResponseAllowed, - ) - if relationErr != nil { - return nil, relationErr - } - tableColumns := inferredRelation.GetColumnDescriptors() - for _, col := range tableColumns { - //nolint:unconvert // defer fix - col := formulation.ColumnDescriptor(col) - relationalType := textStr - schema := col.GetSchema() - if schema != nil && !schema.IsEmpty() && schema.GetType() != "" { - relationalType = dc.GetRelationalType(schema.GetType()) - } - columns = append(columns, typing.NewColDescriptor(col, relationalType)) - relationalColumn := typing.NewRelationalColumn(col.GetName(), relationalType).WithParserNode(col.GetNode()) - relationalTable.PushBackColumn(relationalColumn) - } - } - queryString, err := dc.sqlSystem.GenerateInsertDML(relationalTable, tcc) - if err != nil { - return nil, err - } - priorParameters := tabAnnotated.GetParameters() - // TODO: fix this for dependent table where dependency has `IN` clause!!! - transformedParams, paramErr := util.TransformSQLRawParameters(priorParameters, false) - if paramErr != nil { - return nil, paramErr - } - return NewPreparedStatementCtx( - queryString, - "", - genIDColName, - sessionIDColName, - []string{tableName}, - txnIDColName, - insIDColName, - insEncodedColName, - columns, - 1, - tcc, - nil, - dc.namespaceCollection, - dc.sqlSystem, - transformedParams, - ), - nil -} - -//nolint:revive // future proofing -func (dc *staticDRMConfig) GenerateSelectDML( - tabAnnotated util.AnnotatedTabulation, - prov formulation.Provider, - svc formulation.Service, - resource formulation.Resource, - method formulation.StandardOperationStore, - txnCtrlCtrs internaldto.TxnControlCounters, - selectSuffix, - rewrittenWhere string, - isAwait bool, -) (PreparedStatementCtx, error) { - var quotedColNames []string - var columns []typing.ColumnMetadata - - aliasStr := "" - if tabAnnotated.GetAlias() != "" { - aliasStr = fmt.Sprintf(` AS "%s" `, tabAnnotated.GetAlias()) - } - tn, err := dc.GetCurrentTable(tabAnnotated.GetHeirarchyIdentifiers()) - if err != nil { - return nil, err - } - relationalTable := relationaldto.NewRelationalTable( - tabAnnotated.GetHeirarchyIdentifiers(), - tn.GetDiscoveryID(), - tn.GetName(), - tabAnnotated.GetInputTableName(), - ).WithAlias(aliasStr) - for _, col := range tabAnnotated.GetTabulation().GetColumns() { - //nolint:unconvert,govet // defer fix - col := formulation.ColumnDescriptor(col) - var typeStr string - //nolint:gocritic,exhaustive // TODO: fix - if col.GetSchema() != nil { - typeStr = dc.GetRelationalType(col.GetSchema().GetType()) - } else { - if col.GetVal() != nil { - switch col.GetVal().Type { - case sqlparser.BitVal: - } - } - } - columns = append(columns, typing.NewColDescriptor(col, typeStr)) - // TODO: logic to infer column width - relationalColumn := typing.NewRelationalColumn( - col.GetName(), - typeStr, - ).WithQualifier(col.GetQualifier()).WithParserNode(col.GetNode()) - if col.GetDecoratedCol() == "" { - if col.GetAlias() != "" { - relationalColumn = relationalColumn.WithAlias(col.GetAlias()) - } - } else { - relationalColumn = relationalColumn.WithDecorated(col.GetDecoratedCol()) - } - relationalTable.PushBackColumn(relationalColumn) - quotedColNames = append( //nolint:staticcheck // TODO: fix - quotedColNames, - fmt.Sprintf("%s ", relationalColumn.CanonicalSelectionString()), - ) - } - queryString, err := dc.sqlSystem.GenerateSelectDML(relationalTable, txnCtrlCtrs, selectSuffix, rewrittenWhere) - - if err != nil { - return nil, err - } - - genIDColName := dc.controlAttributes.GetControlGenIDColumnName() - sessionIDColName := dc.controlAttributes.GetControlSsnIDColumnName() - txnIDColName := dc.controlAttributes.GetControlTxnIDColumnName() - insIDColName := dc.controlAttributes.GetControlInsIDColumnName() - return NewPreparedStatementCtx( - queryString, - "", - genIDColName, - sessionIDColName, - nil, - txnIDColName, - insIDColName, - dc.controlAttributes.GetControlInsertEncodedIDColumnName(), - columns, - 1, - txnCtrlCtrs, - nil, - dc.namespaceCollection, - dc.sqlSystem, - tabAnnotated.GetParameters(), - ), nil -} - //nolint:unparam // defer fixing this func (dc *staticDRMConfig) generateControlVarArgs( cp PreparedStatementParameterized, @@ -1018,19 +559,16 @@ func (dc *staticDRMConfig) InsertIntoPhysicalTable( func GenerateDRMConfig( sqlSystem sql_system.SQLSystem, - persistenceSystem formulation.PersistenceSystem, typCfg typing.Config, namespaceCollection tablenamespace.Collection, controlAttributes sqlcontrol.ControlAttributes, ) (Config, error) { rv := &staticDRMConfig{ - namespaceCollection: namespaceCollection, - controlAttributes: controlAttributes, - sqlEngine: sqlSystem.GetSQLEngine(), - sqlSystem: sqlSystem, - typCfg: typCfg, - analyzerFactoryFactory: discovery.NewStandardStaticAnalyzerFactoryFactory(), - persistenceSystem: persistenceSystem, + namespaceCollection: namespaceCollection, + controlAttributes: controlAttributes, + sqlEngine: sqlSystem.GetSQLEngine(), + sqlSystem: sqlSystem, + typCfg: typCfg, } return rv, nil } diff --git a/internal/stackql/entryutil/entryutil.go b/internal/stackql/entryutil/entryutil.go index 832f0be08..2f4264d0c 100644 --- a/internal/stackql/entryutil/entryutil.go +++ b/internal/stackql/entryutil/entryutil.go @@ -7,9 +7,6 @@ import ( "path/filepath" "strings" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/bundle" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" @@ -19,7 +16,10 @@ import ( "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/iqlerror" "github.com/stackql/stackql/internal/stackql/kstore" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/tablenamespace" "github.com/stackql/stackql/internal/stackql/typing" "gopkg.in/yaml.v2" @@ -156,7 +156,10 @@ func buildSQLEngine( sqlCfg dto.SQLBackendCfg, controlAttributes sqlcontrol.ControlAttributes, ) (sqlengine.SQLEngine, error) { - return sqlengine.NewSQLEngine(sqlCfg, controlAttributes) + return sqlengine.NewSQLEngine(sqlengine.Config{ + DBEngine: sqlCfg.DBEngine, DSN: sqlCfg.GetDSN(), DbInitFilePath: sqlCfg.DbInitFilePath, + InitMaxRetries: sqlCfg.InitMaxRetries, InitRetryInitialDelay: sqlCfg.InitRetryInitialDelay, + }, controlAttributes) } func buildGCExec( diff --git a/internal/stackql/execution/mono_valent_execution.go b/internal/stackql/execution/mono_valent_execution.go deleted file mode 100644 index f15a571ce..000000000 --- a/internal/stackql/execution/mono_valent_execution.go +++ /dev/null @@ -1,2081 +0,0 @@ -package execution - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strconv" - "strings" - "time" - - "github.com/stackql/any-sdk/pkg/client" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/httpelement" - "github.com/stackql/any-sdk/pkg/local_template_executor" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/stream_transform" - "github.com/stackql/any-sdk/pkg/streaming" - - "github.com/stackql/any-sdk/pkg/providerinvoker" - "github.com/stackql/any-sdk/public/providerinvokers/anysdkhttp" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/util" - - sdk_internal_dto "github.com/stackql/any-sdk/pkg/internaldto" - "github.com/stackql/any-sdk/public/formulation" -) - -var ( - MonitorPollIntervalSeconds int = 10 //nolint:revive,gochecknoglobals // TODO: global vars refactor -) - -// The naked HTTP status code is not carried through the any-sdk invoker -// boundary (providerinvoker.Result is just {Body, Messages}), so not-found -// detection has to work from upstream error text. That text is NOT uniform: -// any-sdk may publish its own "Status code NNN" prose, a parsed JSON error -// envelope, or a verbatim HTML / XML / plain-text body. Interim measure -// pending naked status code propagation from any-sdk (companion issues: -// any-sdk to expose the status code on the invoker result, stackql to -// consume it and delete this text matching). -// -// upstreamStatusCodeRegex matches any-sdk's structured fragments, eg -// "HTTP response error. Status code 404." and -// "Response error. Status code 404. Body: ...". A sibling of the -// mcpbackend classifier regex; duplicated to keep this package free of -// dependencies on the MCP surface. -var upstreamStatusCodeRegex = regexp.MustCompile(`(?i)status code:?\s*(\d{3})`) //nolint:gochecknoglobals // compiled once - -// upstreamNotFoundLooseRegex matches not-found semantics however an -// unstructured upstream body phrases them: a standalone 404, or the phrase -// "not found", anywhere, any case (eg Go's "404 page not found", nginx's -// "404 Not Found", XML fault strings). A false positive -// here just restores the historical empty-result behaviour, which is the -// safe direction. -var upstreamNotFoundLooseRegex = regexp.MustCompile(`(?i)\b404\b|not\s+found`) //nolint:gochecknoglobals // compiled once - -// isUpstreamNotFound reports whether every upstream error message indicates -// HTTP 404. Querying an absent resource legitimately yields 404 from -// providers and maps onto an empty result set under stackql's database -// semantics, so it is exempt from strict upstream error handling -// (issue #670). When a message carries any-sdk's structured "Status code -// NNN" fragment that number is authoritative (a 403 whose body mentions -// "not found" is still an error); only unstructured text falls back to the -// loose match. -func isUpstreamNotFound(messages []string) bool { - if len(messages) == 0 { - return false - } - for _, message := range messages { - if match := upstreamStatusCodeRegex.FindStringSubmatch(message); match != nil { - if match[1] != "404" { - return false - } - continue - } - if !upstreamNotFoundLooseRegex.MatchString(message) { - return false - } - } - return true -} - -//nolint:gochecknoglobals // TODO: global vars refactor -var ( - _ MonoValentExecutorFactory = (*monoValentExecution)(nil) - nilElisionFunction = func(string, ...any) bool { - return false - } -) - -type MonoValentExecutorFactory interface { - GetExecutor() (func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput, error) -} - -type MonitorMonoValentExecutorFactory interface { - GetMonitorExecutor( - provider formulation.Provider, - op formulation.OperationStore, - precursor primitive.IPrimitive, - initialCtx primitive.IPrimitiveCtx, - comments sqlparser.CommentDirectives, - ) (func(pc primitive.IPrimitiveCtx, bd interface{}) internaldto.ExecutorOutput, error) -} - -// monoValentExecution implements the Builder interface -// and represents the action of acquiring data from an endpoint -// and then persisting that data into a table. -// This data would then subsequently be queried by later execution phases. - -//nolint:unused // TODO: refactor -type monoValentExecution struct { - invoker providerinvoker.Invoker - graphHolder primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - tableMeta tablemetadata.ExtendedTableMetadata - addressSpace formulation.AddressSpace - drmCfg drm.Config - insertPreparedStatementCtx drm.PreparedStatementCtx - insertionContainer tableinsertioncontainer.TableInsertionContainer - txnCtrlCtr internaldto.TxnControlCounters - rowSort func(map[string]map[string]interface{}) []string - root primitivegraph.PrimitiveNode - stream streaming.MapStream - isReadOnly bool //nolint:unused // TODO: build out - isSkipResponse bool - isMutation bool - isAwait bool - defaultHTTPClient *http.Client // for testing purposes only - bldrInput builder_input.BuilderInput -} - -func NewMonoValentExecutorFactory( - graphHolder primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - tableMeta tablemetadata.ExtendedTableMetadata, - insertCtx drm.PreparedStatementCtx, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, - isSkipResponse bool, - isMutation bool, - isAwait bool, - bldrInput builder_input.BuilderInput, -) MonoValentExecutorFactory { - var tcc internaldto.TxnControlCounters - if insertCtx != nil { - tcc = insertCtx.GetGCCtrlCtrs() - } - if stream == nil { - stream = streaming.NewNopMapStream() - } - var defaultHTTPClient *http.Client - if tableMeta != nil { - prov, provErr := tableMeta.GetProvider() - if provErr == nil && prov != nil { - defaultHTTPClient = prov.GetDefaultHTTPClient() - } - } - return &monoValentExecution{ - graphHolder: graphHolder, - handlerCtx: handlerCtx, - tableMeta: tableMeta, - rowSort: rowSort, - drmCfg: handlerCtx.GetDrmConfig(), - insertPreparedStatementCtx: insertCtx, - insertionContainer: insertionContainer, - txnCtrlCtr: tcc, - stream: stream, - isSkipResponse: isSkipResponse, - isMutation: isMutation, - isAwait: isAwait, - defaultHTTPClient: defaultHTTPClient, - invoker: anysdkhttp.New(), - bldrInput: bldrInput, - } -} - -type standardMethodElider struct { - elisionFunc func(string, ...any) bool -} - -func (sme *standardMethodElider) IsElide(reqEncoding string, argz ...any) bool { - return sme.elisionFunc(reqEncoding, argz...) -} - -func NewNilMethodElider() methodElider { - return &standardMethodElider{ - elisionFunc: nilElisionFunction, - } -} - -func NewStandardMethodElider(elisionFunc func(string, ...any) bool) methodElider { - return &standardMethodElider{ - elisionFunc: elisionFunc, - } -} - -//nolint:lll // chaining -func (mv *monoValentExecution) elideActionIfPossible( - currentTcc internaldto.TxnControlCounters, - tableName string, - _ string, // request endocidng placeholder -) methodElider { - elisionFunc := func(reqEncoding string, _ ...any) bool { - olderTcc, isMatch := mv.handlerCtx.GetNamespaceCollection().GetAnalyticsCacheTableNamespaceConfigurator().Match( - tableName, - reqEncoding, - mv.drmCfg.GetControlAttributes().GetControlLatestUpdateColumnName(), mv.drmCfg.GetControlAttributes().GetControlInsertEncodedIDColumnName()) - if isMatch { - nonControlColumns := mv.insertPreparedStatementCtx.GetNonControlColumns() - var nonControlColumnNames []string - for _, c := range nonControlColumns { - nonControlColumnNames = append(nonControlColumnNames, c.GetName()) - } - //nolint:errcheck // TODO: fix - mv.handlerCtx.GetGarbageCollector().Update( - tableName, - olderTcc.Clone(), - currentTcc, - ) - //nolint:errcheck // TODO: fix - mv.insertionContainer.SetTableTxnCounters(tableName, olderTcc) - mv.insertPreparedStatementCtx.SetGCCtrlCtrs(olderTcc) - r, sqlErr := mv.handlerCtx.GetNamespaceCollection().GetAnalyticsCacheTableNamespaceConfigurator().Read( - tableName, reqEncoding, - mv.drmCfg.GetControlAttributes().GetControlInsertEncodedIDColumnName(), - nonControlColumnNames) - if sqlErr != nil { - internaldto.NewErroneousExecutorOutput(sqlErr) - } - mv.drmCfg.ExtractObjectFromSQLRows(r, nonControlColumns, mv.stream) - return true - } - return false - } - return NewStandardMethodElider(elisionFunc) -} - -type methodElider interface { - IsElide(string, ...any) bool -} - -type actionInsertResult struct { - err error - isHousekeepingDone bool -} - -type ActionInsertResult interface { - GetError() (error, bool) - IsHousekeepingDone() bool -} - -//nolint:revive // no idea why this is a thing -func (air *actionInsertResult) GetError() (error, bool) { - return air.err, air.err != nil -} - -func (air *actionInsertResult) IsHousekeepingDone() bool { - return air.isHousekeepingDone -} - -func newActionInsertResult(isHousekeepingDone bool, err error) ActionInsertResult { - return &actionInsertResult{ - err: err, - isHousekeepingDone: isHousekeepingDone, - } -} - -type itemsDTO struct { - items interface{} - ok bool - isNilPayload bool - singletonResponse map[string]interface{} -} - -type ItemisationResult interface { - GetItems() (interface{}, bool) - GetSingltetonResponse() (map[string]interface{}, bool) - IsOk() bool - IsNilPayload() bool -} - -func (id *itemsDTO) GetItems() (interface{}, bool) { - return id.items, id.items != nil -} - -func (id *itemsDTO) GetSingltetonResponse() (map[string]interface{}, bool) { - return id.singletonResponse, id.singletonResponse != nil -} - -func (id *itemsDTO) IsOk() bool { - return id.ok -} - -func (id *itemsDTO) IsNilPayload() bool { - return id.isNilPayload -} - -func newItemisationResult( - items interface{}, - ok bool, - isNilPayload bool, - singletonResponse map[string]interface{}, -) ItemisationResult { - return &itemsDTO{ - items: items, - ok: ok, - isNilPayload: isNilPayload, - singletonResponse: singletonResponse, - } -} - -//nolint:nestif // apathy -func itemise( - target interface{}, - resErr error, - selectItemsKey string, -) ItemisationResult { - var items interface{} - var ok bool - var singletonResponse map[string]interface{} - logging.GetLogger().Infoln(fmt.Sprintf("monoValentExecution.Execute() target = %v", target)) - switch pl := target.(type) { - // add case for xml object, - case map[string]interface{}: - singletonResponse = pl - if selectItemsKey != "" && selectItemsKey != "/*" { - items, ok = pl[selectItemsKey] - if !ok { - if resErr != nil { - items = []interface{}{} - ok = true - } else { - items = []interface{}{ - pl, - } - ok = true - } - } - } else { - items = []interface{}{ - pl, - } - ok = true - } - case []interface{}: - items = pl - ok = true - case []map[string]interface{}: - items = pl - ok = true - case nil: - return newItemisationResult(nil, false, true, singletonResponse) - } - return newItemisationResult(items, ok, false, singletonResponse) -} - -func inferNextPageResponseElement(provider formulation.Provider, method formulation.OperationStore) sdk_internal_dto.HTTPElement { - st, ok := method.GetPaginationResponseTokenSemantic() - if ok { - if tp, err := sdk_internal_dto.ExtractHTTPElement(st.GetLocation()); err == nil { - rv := sdk_internal_dto.NewHTTPElement( - tp, - st.GetKey(), - ) - transformer, tErr := st.GetTransformer() - if tErr == nil && transformer != nil { - rv.SetTransformer(transformer) - } - return rv - } - } - providerStr := provider.GetName() - switch providerStr { - case "github", "okta": - rv := sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.Header, - "Link", - ) - rv.SetTransformer(formulation.DefaultLinkHeaderTransformer) - return rv - default: - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.BodyAttribute, - "nextPageToken", - ) - } -} - -func inferNextPageRequestElement(provider formulation.Provider, method formulation.OperationStore) sdk_internal_dto.HTTPElement { - st, ok := method.GetPaginationRequestTokenSemantic() - if ok { - if tp, err := sdk_internal_dto.ExtractHTTPElement(st.GetLocation()); err == nil { - rv := sdk_internal_dto.NewHTTPElement( - tp, - st.GetKey(), - ) - transformer, tErr := st.GetTransformer() - if tErr == nil && transformer != nil { - rv.SetTransformer(transformer) - } - return rv - } - } - providerStr := provider.GetName() - switch providerStr { - case "github", "okta": - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.RequestString, - "", - ) - default: - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.QueryParam, - "pageToken", - ) - } -} - -type PagingState interface { - GetPageCount() int - IsFinished() bool - GetHTTPResponse() (*http.Response, error) - GetAPIError() error -} - -type httpPagingState struct { - pageCount int - isFinished bool - response client.AnySdkResponse - apiErr error -} - -func (hps *httpPagingState) GetPageCount() int { - return hps.pageCount -} - -func (hps *httpPagingState) IsFinished() bool { - return hps.isFinished -} - -func (hps *httpPagingState) GetHTTPResponse() (*http.Response, error) { - if hps.response != nil { - return hps.response.GetHttpResponse() - } - return nil, fmt.Errorf("nil http response in paging state") -} - -func (hps *httpPagingState) GetAPIError() error { - return hps.apiErr -} - -func newPagingState( - pageCount int, - isFinished bool, - response client.AnySdkResponse, - apiErr error, -) PagingState { - return &httpPagingState{ - pageCount: pageCount, - isFinished: isFinished, - response: response, - apiErr: apiErr, - } -} - -func page( - res formulation.Response, - method formulation.OperationStore, - provider formulation.Provider, - reqCtx formulation.HTTPArmouryParameters, - pageCount int, - rtCtx dto.RuntimeCtx, - authCtx *dto.AuthCtx, - outErrFile io.Writer, - defaultHTTPClient *http.Client, -) PagingState { - npt := inferNextPageResponseElement(provider, method) - nptRequest := inferNextPageRequestElement(provider, method) - if npt == nil || nptRequest == nil { - return newPagingState(pageCount, true, nil, nil) - } - tk := extractNextPageToken(res, npt) - if tk == "" || tk == "" || tk == "[]" || (rtCtx.HTTPPageLimit > 0 && pageCount >= rtCtx.HTTPPageLimit) { - return newPagingState(pageCount, true, nil, nil) - } - pageCount++ - req, reqErr := reqCtx.SetNextPage(method, tk, nptRequest) - if reqErr != nil { - return newPagingState(pageCount, true, nil, reqErr) - } - cc := formulation.NewAnySdkClientConfigurator(rtCtx, provider.GetName(), defaultHTTPClient) - response, apiErr := formulation.CallFromSignature( - cc, rtCtx, authCtx, authCtx.Type, false, outErrFile, provider, - formulation.NewAnySdkOpStoreDesignation(method), - formulation.NewwHTTPAnySdkArgList(req), // TODO: abstract - ) - return newPagingState(pageCount, false, response, apiErr) -} - -type ActionInsertPayload interface { - GetItemisationResult() formulation.ItemisationResult - IsHousekeepingDone() bool - GetTableName() string - GetParamsUsed() map[string]interface{} - GetReqEncoding() string -} - -func newHTTPActionInsertPayload( - itemisationResult formulation.ItemisationResult, - housekeepingDone bool, - tableName string, - paramsUsed map[string]interface{}, - reqEncoding string, -) formulation.ActionInsertPayload { - return formulation.NewActionInsertPayload( - itemisationResult, - housekeepingDone, - tableName, - paramsUsed, - reqEncoding, - ) -} - -//nolint:nestif,gocognit // acceptable for now -func (mv *monoValentExecution) ActionInsertPreparation( - payload formulation.ActionInsertPayload, -) formulation.ActionInsertResult { - itemisationResult := payload.GetItemisationResult() - housekeepingDone := payload.IsHousekeepingDone() - tableName := payload.GetTableName() - paramsUsed := payload.GetParamsUsed() - reqEncoding := payload.GetReqEncoding() - - items, _ := itemisationResult.GetItems() - keys := make(map[string]map[string]interface{}) - iArr, iErr := castItemsArray(items) - if iErr != nil { - return newActionInsertResult(housekeepingDone, iErr) - } - streamErr := mv.stream.Write(iArr) - if streamErr != nil { - return newActionInsertResult(housekeepingDone, streamErr) - } - if len(iArr) > 0 { - if !housekeepingDone && mv.insertPreparedStatementCtx != nil { - _, execErr := mv.handlerCtx.GetSQLEngine().Exec(mv.insertPreparedStatementCtx.GetGCHousekeepingQueries()) - tcc := mv.insertPreparedStatementCtx.GetGCCtrlCtrs() - tcc.SetTableName(tableName) - //nolint:errcheck // TODO: fix - mv.insertionContainer.SetTableTxnCounters(tableName, tcc) - housekeepingDone = true - if execErr != nil { - return newActionInsertResult(housekeepingDone, execErr) - } - } - - for i, item := range iArr { - if item != nil { - if len(paramsUsed) > 0 { - for k, v := range paramsUsed { - if _, itemOk := item[k]; !itemOk { - item[k] = v - } - } - } - - logging.GetLogger().Infoln( - fmt.Sprintf( - "running insert with query = '''%s''', control parameters: %v", - mv.insertPreparedStatementCtx.GetQuery(), - mv.insertPreparedStatementCtx.GetGCCtrlCtrs(), - ), - ) - r, rErr := mv.drmCfg.ExecuteInsertDML( - mv.handlerCtx.GetSQLEngine(), - mv.insertPreparedStatementCtx, - item, - reqEncoding, - ) - logging.GetLogger().Infoln( - fmt.Sprintf( - "insert result = %v, error = %v", - r, - rErr, - ), - ) - if rErr != nil { - expandedErr := fmt.Errorf( - "sql insert error: '%w' from query: %s", - rErr, - mv.insertPreparedStatementCtx.GetQuery(), - ) - return newActionInsertResult(housekeepingDone, expandedErr) - } - keys[strconv.Itoa(i)] = item - } - } - } - - return newActionInsertResult(housekeepingDone, nil) -} - -type AgnosticatePayload interface { - GetArmoury() (formulation.HTTPArmoury, error) - GetProvider() formulation.Provider - GetMethod() formulation.OperationStore - GetTableName() string - GetAuthContext() *dto.AuthCtx - GetRuntimeCtx() dto.RuntimeCtx - GetOutErrFile() io.Writer - GetMaxResultsElement() sdk_internal_dto.HTTPElement - GetElider() methodElider - IsNilResponseAcceptable() bool - GetPolyHandler() PolyHandler - GetSelectItemsKey() string - GetInsertPreparator() formulation.InsertPreparator - IsSkipResponse() bool - IsMutation() bool - IsAwait() bool - GetDefaultHTTPClient() *http.Client // testing purposes only -} - -type httpAgnosticatePayload struct { - tableMeta tablemetadata.ExtendedTableMetadata - provider formulation.Provider - method formulation.OperationStore - tableName string - authCtx *dto.AuthCtx - rtCtx dto.RuntimeCtx - outErrFile io.Writer - maxResultsElement sdk_internal_dto.HTTPElement - elider methodElider - isNilResponseAcceptable bool - polyHandler PolyHandler - selectItemsKey string - insertPreparator formulation.InsertPreparator - isSkipResponse bool - isMutation bool - isAwait bool - defaultHTTPClient *http.Client // testing purposes only -} - -func newHTTPAgnosticatePayload( - tableMeta tablemetadata.ExtendedTableMetadata, - provider formulation.Provider, - method formulation.OperationStore, - tableName string, - authCtx *dto.AuthCtx, - rtCtx dto.RuntimeCtx, - outErrFile io.Writer, - maxResultsElement sdk_internal_dto.HTTPElement, - elider methodElider, - isNilResponseAcceptable bool, - polyHandler PolyHandler, - selectItemsKey string, - insertPreparator formulation.InsertPreparator, - isSkipResponse bool, - isMutation bool, - isAwait bool, - defaultHTTPClient *http.Client, -) AgnosticatePayload { - return &httpAgnosticatePayload{ - tableMeta: tableMeta, - provider: provider, - method: method, - tableName: tableName, - authCtx: authCtx, - rtCtx: rtCtx, - outErrFile: outErrFile, - maxResultsElement: maxResultsElement, - elider: elider, - isNilResponseAcceptable: isNilResponseAcceptable, - polyHandler: polyHandler, - selectItemsKey: selectItemsKey, - insertPreparator: insertPreparator, - isSkipResponse: isSkipResponse, - isMutation: isMutation, - isAwait: isAwait, - defaultHTTPClient: defaultHTTPClient, - } -} - -func (ap *httpAgnosticatePayload) GetPolyHandler() PolyHandler { - return ap.polyHandler -} - -func (ap *httpAgnosticatePayload) GetDefaultHTTPClient() *http.Client { - return ap.defaultHTTPClient -} - -func (ap *httpAgnosticatePayload) IsAwait() bool { - return ap.isAwait -} - -func (ap *httpAgnosticatePayload) GetInsertPreparator() formulation.InsertPreparator { - return ap.insertPreparator -} - -func (ap *httpAgnosticatePayload) GetSelectItemsKey() string { - return ap.selectItemsKey -} - -func (ap *httpAgnosticatePayload) GetArmoury() (formulation.HTTPArmoury, error) { - return ap.tableMeta.GetHTTPArmoury() -} - -func (ap *httpAgnosticatePayload) GetProvider() formulation.Provider { - return ap.provider -} - -func (ap *httpAgnosticatePayload) IsMutation() bool { - return ap.isMutation -} - -func (ap *httpAgnosticatePayload) IsSkipResponse() bool { - return ap.isSkipResponse -} - -func (ap *httpAgnosticatePayload) GetMethod() formulation.OperationStore { - return ap.method -} - -func (ap *httpAgnosticatePayload) GetTableName() string { - return ap.tableName -} - -func (ap *httpAgnosticatePayload) GetAuthContext() *dto.AuthCtx { - return ap.authCtx -} - -func (ap *httpAgnosticatePayload) GetRuntimeCtx() dto.RuntimeCtx { - return ap.rtCtx -} - -func (ap *httpAgnosticatePayload) GetOutErrFile() io.Writer { - return ap.outErrFile -} - -func (ap *httpAgnosticatePayload) GetMaxResultsElement() sdk_internal_dto.HTTPElement { - return ap.maxResultsElement -} - -func (ap *httpAgnosticatePayload) GetElider() methodElider { - return ap.elider -} - -func (ap *httpAgnosticatePayload) IsNilResponseAcceptable() bool { - return ap.isNilResponseAcceptable -} - -type PolyHandler interface { - LogHTTPResponseMap(target interface{}) - MessageHandler([]string) - GetMessages() []string -} - -type standardPolyHandler struct { - handlerCtx handler.HandlerContext - messages []string -} - -func (sph *standardPolyHandler) LogHTTPResponseMap(target interface{}) { - sph.handlerCtx.LogHTTPResponseMap(target) -} - -func (sph *standardPolyHandler) MessageHandler(messages []string) { - sph.messages = append(sph.messages, messages...) -} - -func (sph *standardPolyHandler) GetMessages() []string { - return sph.messages -} - -func NewStandardPolyHandler(handlerCtx handler.HandlerContext) PolyHandler { - return &standardPolyHandler{ - handlerCtx: handlerCtx, - messages: []string{}, - } -} - -func agnosticate( - agPayload AgnosticatePayload, -) (processorResponse, error) { - outErrFile := agPayload.GetOutErrFile() - runtimeCtx := agPayload.GetRuntimeCtx() - provider := agPayload.GetProvider() - tableName := agPayload.GetTableName() - authCtx := agPayload.GetAuthContext() - method := agPayload.GetMethod() - mr := agPayload.GetMaxResultsElement() - elider := agPayload.GetElider() - polyHandler := agPayload.GetPolyHandler() - selectItemsKey := agPayload.GetSelectItemsKey() - insertPreparator := agPayload.GetInsertPreparator() - isSkipResponse := agPayload.IsNilResponseAcceptable() - isMutation := agPayload.IsMutation() - isAwait := agPayload.IsAwait() - defaultHTTPClient := agPayload.GetDefaultHTTPClient() - // TODO: TCC setup - armoury, armouryErr := agPayload.GetArmoury() - if armouryErr != nil { - //nolint:errcheck // TODO: fix - outErrFile.Write([]byte( - fmt.Sprintf( - "error assembling http aspects for resource '%s': %s\n", - method.GetResource().GetID(), - armouryErr.Error(), - ), - ), - ) - return nil, armouryErr - } - if mr != nil { - // TODO: infer param position and act accordingly - ok := true - if ok && runtimeCtx.HTTPMaxResults > 0 { - passOverParams := armoury.GetRequestParams() - for i, p := range passOverParams { - param := p - q := param.GetQuery() - q.Set("maxResults", strconv.Itoa(runtimeCtx.HTTPMaxResults)) - param.SetRawQuery(q.Encode()) - passOverParams[i] = param - } - armoury.SetRequestParams(passOverParams) - } - } - reqParams := armoury.GetRequestParams() - logging.GetLogger().Infof("monoValentExecution.Execute() req param count = %d", len(reqParams)) - var processorResponse processorResponse - for _, rc := range reqParams { - rq := rc - processor := NewProcessor( - NewProcessorPayload( - rq, - elider, - provider, - method, - tableName, - runtimeCtx, - authCtx, - outErrFile, - polyHandler, - selectItemsKey, - insertPreparator, - isSkipResponse, - false, - isAwait, - false, - isMutation, - "", - defaultHTTPClient, - ), - ) - processorResponse = processor.Process() - if processorResponse != nil && processorResponse.GetError() != nil { - return processorResponse, processorResponse.GetError() - } - } - return processorResponse, nil -} - -type ProcessorPayload interface { - GetArmouryParams() formulation.HTTPArmouryParameters - GetElider() methodElider - GetProvider() formulation.Provider - GetMethod() formulation.OperationStore - GetTableName() string - GetRuntimeCtx() dto.RuntimeCtx - GetAuthCtx() *dto.AuthCtx - GetOutErrFile() io.Writer - GetPolyHandler() PolyHandler - GetSelectItemsKey() string - GetInsertPreparator() formulation.InsertPreparator - IsSkipResponse() bool - IsMutation() bool - IsMaterialiseResponse() bool - IsAwait() bool - IsReverseRequired() bool - GetVerb() string - GetDefaultHTTPClient() *http.Client // testing purposes only -} - -func NewProcessorPayload( - armouryParams formulation.HTTPArmouryParameters, - elider methodElider, - provider formulation.Provider, - method formulation.OperationStore, - tableName string, - runtimeCtx dto.RuntimeCtx, - authCtx *dto.AuthCtx, - outErrFile io.Writer, - polyHandler PolyHandler, - selectItemsKey string, - insertPreparator formulation.InsertPreparator, - isSkipResponse bool, - isMaterialiseResponse bool, - isAwait bool, - isReverseRequired bool, - isMutation bool, - verb string, - defaultHTTPClient *http.Client, -) ProcessorPayload { - return &standardProcessorPayload{ - armouryParams: armouryParams, - elider: elider, - provider: provider, - method: method, - tableName: tableName, - runtimeCtx: runtimeCtx, - authCtx: authCtx, - outErrFile: outErrFile, - polyHandler: polyHandler, - selectItemsKey: selectItemsKey, - insertPreparator: insertPreparator, - isSkipResponse: isSkipResponse, - isMaterialiseResponse: isMaterialiseResponse, - isAwait: isAwait, - isReverseRequired: isReverseRequired, - isMutation: isMutation, - verb: verb, - defaultHTTPClient: defaultHTTPClient, - } -} - -type standardProcessorPayload struct { - armouryParams formulation.HTTPArmouryParameters - elider methodElider - provider formulation.Provider - method formulation.OperationStore - tableName string - runtimeCtx dto.RuntimeCtx - authCtx *dto.AuthCtx - outErrFile io.Writer - polyHandler PolyHandler - selectItemsKey string - insertPreparator formulation.InsertPreparator - isSkipResponse bool - isMaterialiseResponse bool - isAwait bool - isReverseRequired bool - isMutation bool - verb string - defaultHTTPClient *http.Client // testing purposes only -} - -func (pp *standardProcessorPayload) GetArmouryParams() formulation.HTTPArmouryParameters { - return pp.armouryParams -} - -func (pp *standardProcessorPayload) IsSkipResponse() bool { - return pp.isSkipResponse -} - -func (pp *standardProcessorPayload) GetDefaultHTTPClient() *http.Client { - return pp.defaultHTTPClient -} - -func (pp *standardProcessorPayload) IsAwait() bool { - return pp.isAwait -} - -func (pp *standardProcessorPayload) IsMutation() bool { - return pp.isMutation -} - -func (pp *standardProcessorPayload) IsReverseRequired() bool { - return pp.isReverseRequired -} - -func (pp *standardProcessorPayload) IsMaterialiseResponse() bool { - return pp.isMaterialiseResponse -} - -func (pp *standardProcessorPayload) GetElider() methodElider { - return pp.elider -} - -func (pp *standardProcessorPayload) GetVerb() string { - if pp.verb == "" { - return "insert" - } - return pp.verb -} - -func (pp *standardProcessorPayload) GetProvider() formulation.Provider { - return pp.provider -} - -func (pp *standardProcessorPayload) GetMethod() formulation.OperationStore { - return pp.method -} - -func (pp *standardProcessorPayload) GetTableName() string { - return pp.tableName -} - -func (pp *standardProcessorPayload) GetRuntimeCtx() dto.RuntimeCtx { - return pp.runtimeCtx -} - -func (pp *standardProcessorPayload) GetAuthCtx() *dto.AuthCtx { - return pp.authCtx -} - -func (pp *standardProcessorPayload) GetOutErrFile() io.Writer { - return pp.outErrFile -} - -func (pp *standardProcessorPayload) GetPolyHandler() PolyHandler { - return pp.polyHandler -} - -func (pp *standardProcessorPayload) GetSelectItemsKey() string { - return pp.selectItemsKey -} - -func (pp *standardProcessorPayload) GetInsertPreparator() formulation.InsertPreparator { - return pp.insertPreparator -} - -type processorResponse interface { - GetError() error - GetSingletonBody() map[string]interface{} - WithSuccessMessages([]string) processorResponse - GetSuccessMessages() []string - AppendReversal(rev formulation.HTTPPreparator) - GetReversalStream() formulation.HttpPreparatorStream - IsFailed() bool - GetFailedMessage() string -} - -type httpProcessorResponse struct { - body map[string]interface{} - err error - successMessages []string - reversalStream formulation.HttpPreparatorStream - isFailed bool - failedMessage string -} - -func (hpr *httpProcessorResponse) IsFailed() bool { - return hpr.isFailed -} - -func (hpr *httpProcessorResponse) GetFailedMessage() string { - return hpr.failedMessage -} - -func (hpr *httpProcessorResponse) WithSuccessMessages(messages []string) processorResponse { - hpr.successMessages = messages - return hpr -} - -//nolint:errcheck // acceptable for now -func (hpr *httpProcessorResponse) AppendReversal(rev formulation.HTTPPreparator) { - hpr.reversalStream.Write(rev) -} - -func (hpr *httpProcessorResponse) GetReversalStream() formulation.HttpPreparatorStream { - return hpr.reversalStream -} - -func (hpr *httpProcessorResponse) GetSuccessMessages() []string { - return hpr.successMessages -} - -func (hpr *httpProcessorResponse) GetError() error { - return hpr.err -} - -func (hpr *httpProcessorResponse) GetSingletonBody() map[string]interface{} { - return hpr.body -} - -func newHTTPProcessorResponse( - body map[string]interface{}, - reversalStream formulation.HttpPreparatorStream, - isFailed bool, - err error, -) processorResponse { - return &httpProcessorResponse{ - body: body, - err: err, - reversalStream: reversalStream, - isFailed: isFailed, - } -} - -type Processor interface { - Process() processorResponse -} - -type standardProcessor struct { - payload ProcessorPayload - defaultHTTPClient *http.Client -} - -func NewProcessor(payload ProcessorPayload) Processor { - return &standardProcessor{ - payload: payload, - defaultHTTPClient: payload.GetDefaultHTTPClient(), - } -} - -//nolint:funlen,bodyclose,gocognit,gocyclo,cyclop // acceptable for now -func (sp *standardProcessor) Process() processorResponse { - processorPayload := sp.payload - armouryParams := processorPayload.GetArmouryParams() - elider := processorPayload.GetElider() - provider := processorPayload.GetProvider() - method := processorPayload.GetMethod() - tableName := processorPayload.GetTableName() - runtimeCtx := processorPayload.GetRuntimeCtx() - authCtx := processorPayload.GetAuthCtx() - outErrFile := processorPayload.GetOutErrFile() - polyHandler := processorPayload.GetPolyHandler() - selectItemsKey := processorPayload.GetSelectItemsKey() - insertPreparator := processorPayload.GetInsertPreparator() - isSkipResponse := processorPayload.IsSkipResponse() - isMutation := processorPayload.IsMutation() - isMaterialiseResponse := processorPayload.IsMaterialiseResponse() - isAwait := processorPayload.IsAwait() - isReverseRequired := processorPayload.IsReverseRequired() - verb := processorPayload.GetVerb() - - reversalStream := formulation.NewHttpPreparatorStream() - - reqCtx := armouryParams - paramsUsed, paramErr := reqCtx.ToFlatMap() - if paramErr != nil { - return newHTTPProcessorResponse(nil, reversalStream, false, paramErr) - } - reqEncoding := reqCtx.Encode() - elideOk := elider.IsElide(reqEncoding) - if elideOk { - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - // TODO: fix cloning ops - cc := formulation.NewAnySdkClientConfigurator(runtimeCtx, provider.GetName(), sp.defaultHTTPClient) - response, apiErr := formulation.CallFromSignature( - cc, - runtimeCtx, - authCtx, - authCtx.Type, - false, - outErrFile, - provider, - formulation.NewAnySdkOpStoreDesignation(method), - reqCtx.GetArgList(), - ) - if response == nil { - if apiErr != nil { - return newHTTPProcessorResponse(nil, reversalStream, false, apiErr) - } - return newHTTPProcessorResponse(nil, reversalStream, false, fmt.Errorf("unacceptable nil response from HTTP call")) - } - //nolint:govet // ignore for now - if isSkipResponse && response == nil { - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - httpResponse, httpResponseErr := response.GetHttpResponse() - if httpResponse != nil && httpResponse.Body != nil { - defer httpResponse.Body.Close() - } - if httpResponse != nil && httpResponse.StatusCode >= 400 && isMaterialiseResponse { - generatedErr := fmt.Errorf("%s over HTTP error: %s", verb, httpResponse.Status) - return newHTTPProcessorResponse(nil, reversalStream, true, generatedErr) - } - // TODO: refactor into package !!TECH_DEBT!! - housekeepingDone := false - nptRequest := inferNextPageRequestElement(provider, method) - pageCount := 1 - for { - if apiErr != nil { - return newHTTPProcessorResponse(nil, reversalStream, false, apiErr) - } - if httpResponseErr != nil { - return newHTTPProcessorResponse(nil, reversalStream, false, httpResponseErr) - } - // TODO: add async monitor here - processed, resErr := method.ProcessResponse(httpResponse) - if resErr != nil { - if isSkipResponse && isMutation && httpResponse.StatusCode < 300 { - return newHTTPProcessorResponse( - nil, reversalStream, false, nil, - ).WithSuccessMessages([]string{"The operation was despatched successfully"}) - } - //nolint:errcheck // TODO: fix - outErrFile.Write( - []byte(fmt.Sprintf("error processing response: %s\n", resErr.Error())), - ) - if processed == nil { - return newHTTPProcessorResponse(nil, reversalStream, false, resErr) - } - } - reversal, reversalExists := processed.GetReversal() - if reversalExists { - reversalAppendErr := reversalStream.Write(reversal) - if reversalAppendErr != nil { - return newHTTPProcessorResponse(nil, reversalStream, false, reversalAppendErr) - } - } - if !reversalExists && isReverseRequired { - return newHTTPProcessorResponse(nil, reversalStream, false, resErr) - } - res, respOk := processed.GetResponse() - if !respOk { - return newHTTPProcessorResponse(nil, reversalStream, false, fmt.Errorf("response is not a valid response")) - } - if res.HasError() { - polyHandler.MessageHandler([]string{res.Error()}) - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - polyHandler.LogHTTPResponseMap(res.GetProcessedBody()) - logging.GetLogger().Infoln(fmt.Sprintf("monoValentExecution.Execute() response = %v", res)) - - if selectItemsKey == "" { - selectItemsKey = method.GetSelectItemsKey() - } - - itemisationResult := itemise(res.GetProcessedBody(), resErr, selectItemsKey) - - if itemisationResult.IsNilPayload() { - break - } - - singletonResponse, hasSingletonResponse := itemisationResult.GetSingltetonResponse() - if isMaterialiseResponse { - msgs := shallowGenerateSuccessMessagesFromHeirarchy(isAwait) - //nolint:gomnd,mnd // acceptable for now - if httpResponse.StatusCode < 300 { - if hasSingletonResponse { - return newHTTPProcessorResponse(singletonResponse, reversalStream, false, nil).WithSuccessMessages(msgs) - } - return newHTTPProcessorResponse(nil, reversalStream, false, nil).WithSuccessMessages(msgs) - } - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - //nolint:gomnd,mnd // acceptable for now - if httpResponse.StatusCode >= 300 { - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - - insertPrepResult := insertPreparator.ActionInsertPreparation( - formulation.NewActionInsertPayload( - itemisationResult, - housekeepingDone, - tableName, - paramsUsed, - reqEncoding, - ), - ) - housekeepingDone = insertPrepResult.IsHousekeepingDone() - insertPrepErr, hasInsertPrepErr := insertPrepResult.GetError() - if !isAwait && isSkipResponse && isMutation && httpResponse.StatusCode < 300 { - return newHTTPProcessorResponse( - nil, reversalStream, false, nil, - ).WithSuccessMessages([]string{"The operation was despatched successfully"}) - } - if hasInsertPrepErr { - return newHTTPProcessorResponse(nil, reversalStream, false, insertPrepErr) - } - - pageResult := page( - res, - method, - provider, - reqCtx, - pageCount, - runtimeCtx, - authCtx, - outErrFile, - sp.defaultHTTPClient, - ) - httpResponse, httpResponseErr = pageResult.GetHTTPResponse() - // if httpResponse != nil && httpResponse.Body != nil { - // defer httpResponse.Body.Close() - // } - if httpResponseErr != nil { - if hasSingletonResponse { // TODO: fix this horrid hack - return newHTTPProcessorResponse(singletonResponse, reversalStream, false, nil) - } - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - // return internaldto.NewErroneousExecutorOutput(httpResponseErr) - } - - if pageResult.IsFinished() { - return newHTTPProcessorResponse(nil, reversalStream, false, nil) - } - - pageCount = pageResult.GetPageCount() - - apiErr = pageResult.GetAPIError() - } - if reqCtx.GetRequest() != nil { - q := reqCtx.GetRequest().URL.Query() - q.Del(nptRequest.GetName()) - reqCtx.SetRawQuery(q.Encode()) - } - return newHTTPProcessorResponse(nil, reversalStream, false, nil) -} - -var ( - _ formulation.BaseArmouryGenerator = (*reconsititutedarmouryGenerator)(nil) -) - -type rawTransformer struct { - node sqlparser.SQLNode - insertValOnlyRows map[int]map[int]interface{} -} - -func (rt *rawTransformer) Transform() (map[int]map[string]interface{}, error) { - return util.ExtractSQLNodeParams(rt.node, rt.insertValOnlyRows) -} - -func newRawTransformer( - node sqlparser.SQLNode, - insertValOnlyRows map[int]map[int]interface{}, -) *rawTransformer { - return &rawTransformer{ - node: node, - insertValOnlyRows: insertValOnlyRows, - } -} - -type reconsititutedarmouryGenerator struct { - prior formulation.BaseArmouryGenerator - updatedParams map[int]map[string]any -} - -func newUpdatedArmouryGenerator( - prior formulation.BaseArmouryGenerator, - updatedParams map[int]map[string]any, -) formulation.BaseArmouryGenerator { - return &reconsititutedarmouryGenerator{ - prior: prior, - updatedParams: updatedParams, - } -} - -func (r *reconsititutedarmouryGenerator) GetHTTPArmoury() (formulation.HTTPArmoury, error) { - if len(r.updatedParams) == 0 { - return r.prior.GetHTTPArmoury() - } - armoury, err := r.prior.GetHTTPArmoury() - if err != nil { - return nil, err - } - updatedArmoury, err := armoury.MergeLateBindingMaps(r.updatedParams) - if err != nil { - return nil, err - } - return updatedArmoury, nil -} - -//nolint:revive,nestif,funlen,gocognit // TODO: investigate -func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput, error) { - prov, err := mv.tableMeta.GetProvider() - if err != nil { - return nil, err - } - provider, providerErr := prov.GetProvider() - if providerErr != nil { - return nil, providerErr - } - m, err := mv.tableMeta.GetMethod() - if err != nil { - return nil, err - } - tableName, err := mv.tableMeta.GetTableName() - if err != nil { - return nil, err - } - authCtx, authCtxErr := mv.handlerCtx.GetAuthContext(prov.GetProviderString()) - if authCtxErr != nil { - return nil, authCtxErr - } - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - requiredDepedencyKey, requiredKeyExists := mv.bldrInput.GetRequiredDataRequestKey() - // lateBindingData := map[int]map[string]any{} - - rawData := map[int]map[int]any{} - if requiredKeyExists { - dataRequest, isRequestExist := mv.graphHolder.GetExecutorOutput(requiredDepedencyKey) - if !isRequestExist { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("required data request not found")) - } - requiredData := dataRequest.GetRawResult() - if requiredData == nil { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("nil required data")) - } - rowCount := 0 - for res, err := requiredData.Read(); res != nil; { - rows, rowErr := res.GetMap() - if rowErr != nil { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("error reading required data: %w", rowErr)) - } - for k, r := range rows { - rawData[k] = r - rowCount++ - } - logging.GetLogger().Infoln(fmt.Sprintf("read %d rows for required data with key '%s'", len(rows), requiredDepedencyKey)) - if err != nil { - if err == io.EOF { - break - } - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("error reading required data: %w", err)) - } - break // yes weird, the api should change - } - if rowCount == 0 { - return internaldto.NewNopEmptyExecutorOutput([]string{"operation skipped due to nil data"}) - } - logging.GetLogger().Infoln(fmt.Sprintf("required data for key '%s' = %v", requiredDepedencyKey, rawData)) - } - - lateBindingData := map[int]map[string]any{} - var lateBindingErr error - if len(rawData) > 0 { - parserNode, _ := mv.bldrInput.GetParserNode() - transformer := newRawTransformer( - parserNode, - rawData, - ) - lateBindingData, lateBindingErr = transformer.Transform() - if lateBindingErr != nil { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("error transforming required data: %w", lateBindingErr)) - } - logging.GetLogger().Infoln(fmt.Sprintf("late binding data = %v", lateBindingData)) - } - armouryGenerator := newUpdatedArmouryGenerator( - mv.tableMeta, - lateBindingData, - ) - currentTcc := mv.insertPreparedStatementCtx.GetGCCtrlCtrs().Clone() - mv.graphHolder.AddTxnControlCounters(currentTcc) - mr := prov.InferMaxResultsElement(m) - polyHandler := NewStandardPolyHandler( - mv.handlerCtx, - ) - protocolType, protocolTypeErr := provider.GetProtocolType() - if protocolTypeErr != nil { - return internaldto.NewErroneousExecutorOutput(protocolTypeErr) - } - //nolint:exhaustive // acceptable for now - switch protocolType { - case client.LocalTemplated: - inlines := m.GetInline() - if len(inlines) == 0 { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("no inlines found")) - } - executor := local_template_executor.NewLocalTemplateExecutor( - inlines[0], - inlines[1:], - nil, - ) - armoury, armouryErr := armouryGenerator.GetHTTPArmoury() - if armouryErr != nil { - return internaldto.NewErroneousExecutorOutput(armouryErr) - } - requestParams := armoury.GetRequestParams() - logging.GetLogger().Infoln(fmt.Sprintf("requestParams = %v", requestParams)) - flatInlineParams := make(map[string]interface{}) - for _, p := range requestParams { - foundInlineParams, loopErr := p.GetParameters().GetInlineParameterFlatMap() - if loopErr == nil { - flatInlineParams = foundInlineParams - } - break //nolint:staticcheck // acceptable for now - } - // if mapsErr != nil { - // return internaldto.NewErroneousExecutorOutput(mapsErr) - // } - // paramMap := interestingMaps.getParameterMap() - // params := paramMap[0] - resp, exErr := executor.Execute( - map[string]any{"parameters": flatInlineParams}, - ) - if exErr != nil { - return internaldto.NewErroneousExecutorOutput(exErr) - } - var backendMessages []string - stdOut, stdOutExists := resp.GetStdOut() - var stdoutStr string - if stdOutExists { - stdoutStr = stdOut.String() - expectedResponse, isExpectedResponse := m.GetResponse() - if isExpectedResponse { - responseTransform, responseTransformExists := expectedResponse.GetTransform() - if responseTransformExists { - input := stdoutStr - streamTransformerFactory := stream_transform.NewStreamTransformerFactory( - responseTransform.GetType(), - responseTransform.GetBody(), - ) - if !streamTransformerFactory.IsTransformable() { - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("unsupported template type: %s", responseTransform.GetType()), - ) - } - tfm, getTfmErr := streamTransformerFactory.GetTransformer(input) - if getTfmErr != nil { - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("failed to transform: %w", getTfmErr)) - } - transformError := tfm.Transform() - if transformError != nil { - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("failed to transform: %w", transformError)) - } - outStream := tfm.GetOutStream() - outputBytes, readErr := io.ReadAll(outStream) - if readErr != nil { - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("failed to read transformed stream: %w", readErr)) - } - outputStr := string(outputBytes) - stdoutStr = outputStr - } - } - var res []map[string]interface{} - resErr := json.Unmarshal([]byte(stdoutStr), &res) - itemisationResult := itemise(res, resErr, "") - insertPrepResult := mv.ActionInsertPreparation( - newHTTPActionInsertPayload( - itemisationResult, - false, - tableName, - flatInlineParams, - "", - ), - ) - insertErr, hasErr := insertPrepResult.GetError() - if hasErr { - return internaldto.NewErroneousExecutorOutput(insertErr) - } - // fmt.Fprintf(os.Stdout, "%s", stdoutStr) - } - // if stdOutExists { - // backendMessages = append(backendMessages, stdOut.String()) - // } - stdErr, stdErrExists := resp.GetStdErr() - if stdErrExists { - backendMessages = append(backendMessages, stdErr.String()) - } - backendMessages = append(backendMessages, "OK") - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - internaldto.NewBackendMessages(backendMessages), - nil, - ) - case client.HTTP: - invRes, invErr := mv.invoker.Invoke(context.Background(), providerinvoker.Request{ - Payload: formulation.NewPayload( - armouryGenerator, - provider, - m, - tableName, - authCtx, - mv.handlerCtx.GetRuntimeContext(), - mv.handlerCtx.GetOutErrFile(), - mr, - mv.elideActionIfPossible( - currentTcc, - tableName, - "", // late binding, should remove AOT reference - ), - true, - polyHandler, - mv.tableMeta.GetSelectItemsKey(), - mv, - mv.isSkipResponse, - mv.isMutation, - mv.isAwait, - mv.defaultHTTPClient, - mv.handlerCtx, - ), - }) - if invErr != nil { - return internaldto.NewErroneousExecutorOutput(invErr) - } - var castMessages internaldto.BackendMessages - if len(invRes.Messages) > 0 { - castMessages = internaldto.NewBackendMessages(invRes.Messages) - } - // On the data acquisition (SELECT-shaped) path the any-sdk - // invoker only attaches messages when the upstream response was - // erroneous (eg HTTP 4xx / 5xx with a parseable error body, in - // which case the parsed error body may also be returned as a - // singleton Body); success messages are exclusive to mutation - // and skip-response flows. Under strict upstream error - // semantics (the MCP server, issue #670) that must surface as a - // statement error so callers can distinguish "zero rows" from - // "query failed". HTTP 404 is exempt: stackql presents provider - // resources as tables, so "not found" is the database-semantics - // equivalent of zero rows and keeps the historical empty-result - // behaviour everywhere, MCP included. - if mv.handlerCtx.IsStrictUpstreamErrors() && - !mv.isMutation && !mv.isSkipResponse && len(invRes.Messages) > 0 && - !isUpstreamNotFound(invRes.Messages) { - return internaldto.NewExecutorOutput( - nil, nil, nil, castMessages, - fmt.Errorf("upstream provider error: %s", strings.Join(invRes.Messages, "; ")), - ) - } - if invRes.Body == nil { - return internaldto.NewExecutorOutput(nil, nil, nil, castMessages, nil) - } - bdyMap, bodyMapOk := invRes.Body.(map[string]interface{}) - if !bodyMapOk { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("unexpected body type '%T'", invRes.Body)) - } - return internaldto.NewExecutorOutput(nil, bdyMap, nil, castMessages, err) - default: - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("unsupported protocol type '%v'", protocolType)) - } - } - return ex, nil -} - -func shimProcessHTTP( - url string, - rtCtx dto.RuntimeCtx, - authCtx *dto.AuthCtx, - provider formulation.Provider, - m formulation.OperationStore, - outErrFile io.Writer, -) (*http.Response, error) { - req, monitorReqErr := formulation.GetMonitorRequest(url) - if monitorReqErr != nil { - return nil, monitorReqErr - } - cc := formulation.NewAnySdkClientConfigurator(rtCtx, provider.GetName(), nil) - anySdkResponse, apiErr := formulation.CallFromSignature( - cc, rtCtx, authCtx, authCtx.Type, false, outErrFile, provider, formulation.NewAnySdkOpStoreDesignation(m), req) - - if apiErr != nil { - return nil, apiErr - } - httpResponse, httpResponseErr := anySdkResponse.GetHttpResponse() - if httpResponseErr != nil { - return nil, httpResponseErr - } - return httpResponse, nil -} - -//nolint:funlen,gocognit,errcheck // acceptable for now -func GetMonitorExecutor( - handlerCtx handler.HandlerContext, - provider formulation.Provider, - op formulation.OperationStore, - precursor primitive.IPrimitive, - initialCtx primitive.IPrimitiveCtx, - comments sqlparser.CommentDirectives, - isReturning bool, - insertCtx drm.PreparedStatementCtx, - drmCfg drm.Config, -) (primitive.IPrimitive, error) { - m := op - // tableName, err := mv.tableMeta.GetTableName() - // if err != nil { - // return nil, err - // } - // authCtx, authCtxErr := mv.handlerCtx.GetAuthContext(prov.GetProviderString()) - // if authCtxErr != nil { - // return nil, authCtxErr - // } - asyncPrim := asyncHTTPMonitorPrimitive{ - handlerCtx: handlerCtx, - prov: provider, - op: op, - initialCtx: initialCtx, - precursor: precursor, - elapsedSeconds: 0, - pollIntervalSeconds: MonitorPollIntervalSeconds, - comments: comments, - insertCtx: insertCtx, - drmCfg: drmCfg, - } - if comments != nil { - asyncPrim.noStatus = comments.IsSet("NOSTATUS") - } - rtCtx := handlerCtx.GetRuntimeContext() - outErrFile := handlerCtx.GetOutErrFile() - asyncPrim.executor = func(pc primitive.IPrimitiveCtx, bd interface{}) internaldto.ExecutorOutput { - body, ok := bd.(map[string]interface{}) - if !ok { - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - nil, - fmt.Errorf("cannot execute monitor: response body of type '%T' unreadable", bd), - ) - } - if pc == nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, fmt.Errorf("cannot execute monitor: nil plan primitive")) - } - if body == nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, fmt.Errorf("cannot execute monitor: no body present")) - } - logging.GetLogger().Infoln(fmt.Sprintf("body = %v", body)) - - operationDescriptor := getOpDescriptor(body) - endTime, endTimeOk := body["endTime"] - prStr := provider.GetName() - //nolint:nestif // acceptable for now - if endTimeOk && endTime != "" { - targetLink, targetLinkOK := body["targetLink"] - if targetLinkOK && isReturning { - authCtx, authErr := pc.GetAuthContext(prStr) - if authErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, authErr) - } - if authCtx == nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, fmt.Errorf("cannot execute monitor: no auth context")) - } - targetLinkStr, targetLinkStrOk := targetLink.(string) - if !targetLinkStrOk { - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - nil, - fmt.Errorf("cannot execute monitor: 'targetLink' is not a string"), - ) - } - httpResponse, httpResponseErr := shimProcessHTTP( - targetLinkStr, - rtCtx, - authCtx, - provider, - m, - outErrFile, - ) - if httpResponseErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, httpResponseErr) - } - - if httpResponse != nil && httpResponse.Body != nil { - defer httpResponse.Body.Close() - } - target, targetErr := m.DeprecatedProcessResponse(httpResponse) - handlerCtx.LogHTTPResponseMap(target) - if targetErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, targetErr) - } - // TODO: insert into table here - if isReturning { - if asyncPrim.insertCtx != nil { - _, rErr := asyncPrim.drmCfg.ExecuteInsertDML( - handlerCtx.GetSQLEngine(), - asyncPrim.insertCtx, - target, - "", // TODO: figure out how on earth to compute this encoding - ) - if rErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, rErr) - } - } - } - return prepareResultSet(&asyncPrim, pc, target, operationDescriptor) - } - return prepareResultSet(&asyncPrim, pc, body, operationDescriptor) - } - url, ok := body["selfLink"] - if !ok { - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - nil, - fmt.Errorf("cannot execute monitor: no 'selfLink' property present"), - ) - } - authCtx, authErr := pc.GetAuthContext(prStr) - if authErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, authErr) - } - if authCtx == nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, fmt.Errorf("cannot execute monitor: no auth context")) - } - time.Sleep(time.Duration(asyncPrim.pollIntervalSeconds) * time.Second) - asyncPrim.elapsedSeconds += asyncPrim.pollIntervalSeconds - if !asyncPrim.noStatus { - //nolint:errcheck //TODO: handle error - pc.GetErrWriter().Write( - []byte( - fmt.Sprintf( - "%s in progress, %d seconds elapsed", - operationDescriptor, - asyncPrim.elapsedSeconds, - ) + fmt.Sprintln(""), - ), - ) - } - req, monitorReqErr := formulation.GetMonitorRequest(url.(string)) - if monitorReqErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, monitorReqErr) - } - cc := formulation.NewAnySdkClientConfigurator(rtCtx, provider.GetName(), nil) - anySdkResponse, apiErr := formulation.CallFromSignature( - cc, rtCtx, authCtx, authCtx.Type, false, outErrFile, provider, formulation.NewAnySdkOpStoreDesignation(m), req) - - if apiErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, apiErr) - } - httpResponse, httpResponseErr := anySdkResponse.GetHttpResponse() - if httpResponseErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, httpResponseErr) - } - - if httpResponse != nil && httpResponse.Body != nil { - defer httpResponse.Body.Close() - } - target, targetErr := m.DeprecatedProcessResponse(httpResponse) - handlerCtx.LogHTTPResponseMap(target) - if targetErr != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, targetErr) - } - return asyncPrim.executor(internaldto.NewBasicPrimitiveContext( - pc.GetAuthContext, - pc.GetWriter(), - pc.GetErrWriter(), - ), - target) - } - return &asyncPrim, nil -} - -func extractNextPageToken(res formulation.Response, tokenKey sdk_internal_dto.HTTPElement) string { - //nolint:exhaustive // TODO: review - switch tokenKey.GetType() { - case sdk_internal_dto.BodyAttribute: - return extractNextPageTokenFromBody(res, tokenKey) - case sdk_internal_dto.Header: - return extractNextPageTokenFromHeader(res, tokenKey) - } - return "" -} - -//nolint:bodyclose // acceptable for now -func extractNextPageTokenFromHeader(res formulation.Response, tokenKey sdk_internal_dto.HTTPElement) string { - r := res.GetHttpResponse() - if r == nil { - return "" - } - header := r.Header - if tokenKey.IsTransformerPresent() { - tf, err := tokenKey.Transformer(header) - if err != nil { - return "" - } - rv, ok := tf.(string) - if !ok { - return "" - } - return rv - } - vals := header.Values(tokenKey.GetName()) - if len(vals) == 1 { - return vals[0] - } - return "" -} - -func extractNextPageTokenFromBody(res formulation.Response, tokenKey sdk_internal_dto.HTTPElement) string { - elem, err := httpelement.NewHTTPElement(tokenKey.GetName(), "body") - if err == nil { - rawVal, rawErr := res.ExtractElement(elem) - if rawErr == nil { - switch v := rawVal.(type) { - case []interface{}: - if len(v) == 1 { - return fmt.Sprintf("%v", v[0]) - } - default: - return fmt.Sprintf("%v", v) - } - } - } - body := res.GetProcessedBody() - switch target := body.(type) { //nolint:gocritic // TODO: review - case map[string]interface{}: - tokenName := tokenKey.GetName() - nextPageToken, ok := target[tokenName] - if !ok || nextPageToken == "" { - logging.GetLogger().Infoln("breaking out") - return "" - } - tk, ok := nextPageToken.(string) - if !ok { - logging.GetLogger().Infoln("breaking out") - return "" - } - return tk - } - return "" -} - -type asyncHTTPMonitorPrimitive struct { - handlerCtx handler.HandlerContext - prov formulation.Provider - op formulation.OperationStore - initialCtx primitive.IPrimitiveCtx - precursor primitive.IPrimitive - executor func(pc primitive.IPrimitiveCtx, initalBody interface{}) internaldto.ExecutorOutput - elapsedSeconds int - pollIntervalSeconds int - noStatus bool - id int64 - comments sqlparser.CommentDirectives - insertCtx drm.PreparedStatementCtx - drmCfg drm.Config -} - -func (pr *asyncHTTPMonitorPrimitive) SetTxnID(_ int) { -} - -func (pr *asyncHTTPMonitorPrimitive) IsReadOnly() bool { - return false -} - -func (pr *asyncHTTPMonitorPrimitive) GetRedoLog() (binlog.LogEntry, bool) { - return nil, false -} - -func (pr *asyncHTTPMonitorPrimitive) GetUndoLog() (binlog.LogEntry, bool) { - return nil, false -} - -func (pr *asyncHTTPMonitorPrimitive) WithDebugName(_ string) primitive.IPrimitive { - return pr -} - -func (pr *asyncHTTPMonitorPrimitive) SetUndoLog(_ binlog.LogEntry) { -} - -func (pr *asyncHTTPMonitorPrimitive) SetRedoLog(_ binlog.LogEntry) { -} - -func (pr *asyncHTTPMonitorPrimitive) IncidentData(fromID int64, input internaldto.ExecutorOutput) error { - return pr.precursor.IncidentData(fromID, input) -} - -func (pr *asyncHTTPMonitorPrimitive) SetInputAlias(alias string, id int64) error { - return pr.precursor.SetInputAlias(alias, id) -} - -func (pr *asyncHTTPMonitorPrimitive) Optimise() error { - return nil -} - -func (pr *asyncHTTPMonitorPrimitive) Execute(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - if pr.executor != nil { - if pc == nil { - pc = pr.initialCtx - } - subPr := pr.precursor.Execute(pc) - if subPr.GetError() != nil || pr.executor == nil { - return subPr - } - prStr := pr.prov.GetName() - // seems pointless - _, err := pr.initialCtx.GetAuthContext(prStr) - if err != nil { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) - } - // - asyP := internaldto.NewBasicPrimitiveContext( - pr.initialCtx.GetAuthContext, - pc.GetWriter(), - pc.GetErrWriter(), - ) - return pr.executor(asyP, subPr.GetOutputBody()) - } - return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) -} - -func (pr *asyncHTTPMonitorPrimitive) ID() int64 { - return pr.id -} - -func (pr *asyncHTTPMonitorPrimitive) GetInputFromAlias(string) (internaldto.ExecutorOutput, bool) { - var rv internaldto.ExecutorOutput - return rv, false -} - -func (pr *asyncHTTPMonitorPrimitive) SetExecutor(_ func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput) error { - return fmt.Errorf("asyncHTTPMonitorPrimitive does not support SetExecutor()") -} - -func getOpDescriptor(body map[string]interface{}) string { - operationDescriptor := "operation" - if body == nil { - return operationDescriptor - } - //nolint:nestif,govet // TODO: refactor - if descriptor, ok := body["kind"]; ok { - if descriptorStr, ok := descriptor.(string); ok { - operationDescriptor = descriptorStr - if typeElem, ok := body["operationType"]; ok { - if typeStr, ok := typeElem.(string); ok { - operationDescriptor = fmt.Sprintf("%s: %s", descriptorStr, typeStr) - } - } - } - } - return operationDescriptor -} - -func prepareResultSet( - prim *asyncHTTPMonitorPrimitive, - pc primitive.IPrimitiveCtx, - target map[string]interface{}, - operationDescriptor string, -) internaldto.ExecutorOutput { - payload := internaldto.PrepareResultSetDTO{ - OutputBody: target, - Msg: nil, - RowMap: nil, - ColumnOrder: nil, - RowSort: nil, - Err: nil, - } - if !prim.noStatus { - //nolint:errcheck //TODO: handle error - pc.GetErrWriter().Write([]byte(fmt.Sprintf("%s complete", operationDescriptor) + fmt.Sprintln(""))) - } - return util.PrepareResultSet(payload) -} - -func castItemsArray(iArr interface{}) ([]map[string]interface{}, error) { - switch iArr := iArr.(type) { - case []map[string]interface{}: - return iArr, nil - case []interface{}: - var rv []map[string]interface{} - for i := range iArr { - item, ok := iArr[i].(map[string]interface{}) - if !ok { - if iArr[i] != nil { - item = map[string]interface{}{formulation.AnonymousColumnName: iArr[i]} - } else { - item = nil - } - } - rv = append(rv, item) - } - return rv, nil - default: - return nil, fmt.Errorf("cannot accept items array of type = '%T'", iArr) - } -} - -func shallowGenerateSuccessMessagesFromHeirarchy(isAwait bool) []string { - baseSuccessString := "The operation completed successfully" - if !isAwait { - baseSuccessString = "The operation was despatched successfully" - } - successMsgs := []string{ - baseSuccessString, - } - return successMsgs -} diff --git a/internal/stackql/execution/upstream_not_found_test.go b/internal/stackql/execution/upstream_not_found_test.go deleted file mode 100644 index 88ee877cd..000000000 --- a/internal/stackql/execution/upstream_not_found_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package execution //nolint:testpackage // exercise unexported helper - -import "testing" - -func TestIsUpstreamNotFound(t *testing.T) { - cases := []struct { - name string - messages []string - want bool - }{ - { - name: "single 404 detail form", - messages: []string{"HTTP response error. Status code 404. Detail: 'not found'"}, - want: true, - }, - { - name: "single 404 body form", - messages: []string{`Response error. Status code 404. Body: {"message":"Not Found"}`}, - want: true, - }, - { - name: "403 is not absence", - messages: []string{"Response error. Status code 403. Body: {}"}, - want: false, - }, - { - name: "mixed 404 and 500 is not absence", - messages: []string{ - "Response error. Status code 404. Body: {}", - "Response error. Status code 500. Body: {}", - }, - want: false, - }, - { - name: "no status code fragment and no not-found phrasing", - messages: []string{"something else entirely"}, - want: false, - }, - { - name: "empty messages", - messages: nil, - want: false, - }, - { - name: "unstructured go default body", - messages: []string{"404 page not found"}, - want: true, - }, - { - name: "unstructured html body", - messages: []string{"404 Not Found

NOT FOUND

"}, - want: true, - }, - { - name: "unstructured xml fault", - messages: []string{"404The specified key does not exist."}, - want: true, - }, - { - name: "unstructured mixed case phrase only", - messages: []string{"the requested thing was Not FOUND on this server"}, - want: true, - }, - { - name: "structured 403 with not-found wording in body is still an error", - messages: []string{`Response error. Status code 403. Body: {"message":"secret not found for caller"}`}, - want: false, - }, - { - name: "digits embedded in larger number are not a 404", - messages: []string{"upstream failure, trace id 14042"}, - want: false, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := isUpstreamNotFound(tc.messages); got != tc.want { - t.Errorf("isUpstreamNotFound(%v) = %t, want %t", tc.messages, got, tc.want) - } - }) - } -} diff --git a/internal/stackql/garbagecollector/garbagecollector.go b/internal/stackql/garbagecollector/garbagecollector.go index 7e74a7773..3cf696652 100644 --- a/internal/stackql/garbagecollector/garbagecollector.go +++ b/internal/stackql/garbagecollector/garbagecollector.go @@ -1,10 +1,10 @@ package garbagecollector import ( - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/gcexec" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/sqlengine" ) var ( diff --git a/internal/stackql/garbagecollector/garbagecollector_test.go b/internal/stackql/garbagecollector/garbagecollector_test.go index dd8a3b4ca..70d51c8fc 100644 --- a/internal/stackql/garbagecollector/garbagecollector_test.go +++ b/internal/stackql/garbagecollector/garbagecollector_test.go @@ -3,8 +3,8 @@ package garbagecollector //nolint:testpackage // to test unexported methods import ( "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/internal/stackql/gcexec/gcexec.go b/internal/stackql/gcexec/gcexec.go index ae53a43c5..0249f90df 100644 --- a/internal/stackql/gcexec/gcexec.go +++ b/internal/stackql/gcexec/gcexec.go @@ -3,10 +3,10 @@ package gcexec import ( "sync" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/kstore" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/tablenamespace" ) diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index 4abc0ae66..80fdbf397 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -11,25 +11,26 @@ import ( "sync" "time" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/jsonpath" - "github.com/stackql/any-sdk/pkg/netutils" - "github.com/stackql/any-sdk/pkg/nomenclature" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql/stackql/internal/stackql/acid/tsm" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/bundle" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/dbmsinternal" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/garbagecollector" "github.com/stackql/stackql/internal/stackql/intrinsic" + "github.com/stackql/stackql/internal/stackql/jsonpath" "github.com/stackql/stackql/internal/stackql/kstore" + "github.com/stackql/stackql/internal/stackql/netutils" + "github.com/stackql/stackql/internal/stackql/nomenclature" "github.com/stackql/stackql/internal/stackql/provider" + "github.com/stackql/stackql/internal/stackql/registry" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/tablenamespace" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/internal/stackql/writer" @@ -65,7 +66,7 @@ type HandlerContext interface { //nolint:revive // don't mind stuttering this on GetControlAttributes() sqlcontrol.ControlAttributes GetCurrentProvider() string GetAuthContexts() dto.AuthContexts - GetRegistry() formulation.RegistryAPI + GetRegistry() registry.RegistryAPI GetErrorPresentation() string GetOutfile() io.Writer GetOutErrFile() io.Writer @@ -73,7 +74,6 @@ type HandlerContext interface { //nolint:revive // don't mind stuttering this on GetSQLDataSource(name string) (sql_datasource.SQLDataSource, bool) GetSQLEngine() sqlengine.SQLEngine GetSQLSystem() sql_system.SQLSystem - GetPersistenceSystem() formulation.PersistenceSystem GetGarbageCollector() garbagecollector.GarbageCollector GetDrmConfig() drm.Config SetTxnCounterMgr(txncounter.Manager) @@ -138,14 +138,13 @@ type standardHandlerContext struct { authContexts dto.AuthContexts docAuthProviders map[string]struct{} sqlDataSources map[string]sql_datasource.SQLDataSource - registry formulation.RegistryAPI + registry registry.RegistryAPI errorPresentation string outfile io.Writer outErrFile io.Writer lRUCache *lrucache.LRUCache sqlEngine sqlengine.SQLEngine sqlSystem sql_system.SQLSystem - persistenceSystem formulation.PersistenceSystem garbageCollector garbagecollector.GarbageCollector drmConfig drm.Config txnCounterMgr txncounter.Manager @@ -248,9 +247,9 @@ func (hc *standardHandlerContext) GetAuthContexts() dto.AuthContexts { return hc.authContexts } -func (hc *standardHandlerContext) GetRegistry() formulation.RegistryAPI { return hc.registry } -func (hc *standardHandlerContext) GetErrorPresentation() string { return hc.errorPresentation } -func (hc *standardHandlerContext) GetOutfile() io.Writer { return hc.outfile } +func (hc *standardHandlerContext) GetRegistry() registry.RegistryAPI { return hc.registry } +func (hc *standardHandlerContext) GetErrorPresentation() string { return hc.errorPresentation } +func (hc *standardHandlerContext) GetOutfile() io.Writer { return hc.outfile } func (hc *standardHandlerContext) GetOutErrFile() io.Writer { defer hc.sessionCtxMutex.Unlock() hc.sessionCtxMutex.Lock() @@ -264,9 +263,7 @@ func (hc *standardHandlerContext) SetOutErrFile(w io.Writer) { func (hc *standardHandlerContext) GetLRUCache() *lrucache.LRUCache { return hc.lRUCache } func (hc *standardHandlerContext) GetSQLEngine() sqlengine.SQLEngine { return hc.sqlEngine } func (hc *standardHandlerContext) GetSQLSystem() sql_system.SQLSystem { return hc.sqlSystem } -func (hc *standardHandlerContext) GetPersistenceSystem() formulation.PersistenceSystem { - return hc.persistenceSystem -} + func (hc *standardHandlerContext) GetGarbageCollector() garbagecollector.GarbageCollector { return hc.garbageCollector } @@ -284,8 +281,8 @@ func (hc *standardHandlerContext) GetPGInternalRouter() dbmsinternal.Router { return hc.pgInternalRouter } -func getProviderMap(providerName string, providerDesc formulation.ProviderDescription) map[string]interface{} { - latestVersion, err := providerDesc.GetLatestVersion() +func getProviderMap(providerName string, providerDesc registry.ProviderDescription) map[string]interface{} { + latestVersion, err := registry.LatestVersion(providerDesc.Versions) if err != nil { //nolint:lll // long message latestVersion = fmt.Sprintf("could not infer latest version due to error. Suggested action is that you wipe the local provider directory. Error = '%s'", err.Error()) @@ -299,7 +296,7 @@ func getProviderMap(providerName string, providerDesc formulation.ProviderDescri func getProviderMapExtended( providerName string, - providerDesc formulation.ProviderDescription, + providerDesc registry.ProviderDescription, ) map[string]interface{} { return getProviderMap(providerName, providerDesc) } @@ -388,8 +385,7 @@ func (hc *standardHandlerContext) GetProvider(providerName string) (provider.IPr if !ok { prov, err = provider.GenerateProvider( hc.runtimeContext, ds.Name, ds.Tag, - hc.registry, hc.sqlSystem, hc.persistenceSystem, - hc.defaultHTTPClient, + hc.registry, hc.sqlSystem, ) if err == nil { hc.providers[providerName] = prov @@ -459,17 +455,25 @@ func (hc *standardHandlerContext) LogHTTPResponseMap(target interface{}) { } func (hc *standardHandlerContext) GetAuthContext(providerName string) (*dto.AuthCtx, error) { - var err error if providerName == "" { providerName = hc.runtimeContext.ProviderStr } hc.authMapMutex.Lock() - defer hc.authMapMutex.Unlock() authCtx, ok := hc.authContexts[providerName] - if !ok { - err = fmt.Errorf("cannot find AUTH context for provider = '%s'", providerName) + hc.authMapMutex.Unlock() + if ok { + return authCtx, nil + } + // Load only registration defaults. Authentication itself belongs to Omni. + if _, err := hc.GetProvider(providerName); err != nil { + return nil, err } - return authCtx, err + hc.authMapMutex.Lock() + defer hc.authMapMutex.Unlock() + if authCtx, ok = hc.authContexts[providerName]; ok { + return authCtx, nil + } + return nil, fmt.Errorf("cannot find AUTH context for provider = '%s'", providerName) } func (hc *standardHandlerContext) UpdateIsolationLevel(isolationLevelStr string) error { @@ -569,12 +573,12 @@ func (hc *standardHandlerContext) GetDBMSInternalRouter() dbmsinternal.Router { return hc.pgInternalRouter } -func GetRegistry(runtimeCtx dto.RuntimeCtx) (formulation.RegistryAPI, error) { +func GetRegistry(runtimeCtx dto.RuntimeCtx) (registry.RegistryAPI, error) { return getRegistry(runtimeCtx) } -func getRegistry(runtimeCtx dto.RuntimeCtx) (formulation.RegistryAPI, error) { - var rc formulation.RegistryConfig +func getRegistry(runtimeCtx dto.RuntimeCtx) (registry.RegistryAPI, error) { + var rc omnisdk.RegistryConfig err := yaml.Unmarshal([]byte(runtimeCtx.RegistryRaw), &rc) if err != nil { return nil, err @@ -587,7 +591,7 @@ func getRegistry(runtimeCtx dto.RuntimeCtx) (formulation.RegistryAPI, error) { } } rt := netutils.GetRoundTripper(runtimeCtx, nil) - return formulation.NewRegistry(rc, rt) + return registry.NewRegistry(rc, rt) } func (hc *standardHandlerContext) Clone() HandlerContext { @@ -612,7 +616,6 @@ func (hc *standardHandlerContext) Clone() HandlerContext { sqlEngine: hc.sqlEngine, sqlDataSources: hc.sqlDataSources, sqlSystem: hc.sqlSystem, - persistenceSystem: hc.persistenceSystem, garbageCollector: hc.garbageCollector, outErrFile: hc.outErrFile, outfile: hc.outfile, @@ -693,7 +696,6 @@ func NewHandlerCtx( sqlEngine: sqlEngine, sqlDataSources: inputBundle.GetSQLDataSources(), sqlSystem: inputBundle.GetSQLSystem(), - persistenceSystem: inputBundle.GetPersistenceSystem(), garbageCollector: inputBundle.GetGC(), txnCounterMgr: inputBundle.GetTxnCounterManager(), txnStore: inputBundle.GetTxnStore(), @@ -711,7 +713,6 @@ func NewHandlerCtx( } drmCfg, err := drm.GenerateDRMConfig( inputBundle.GetSQLSystem(), - inputBundle.GetPersistenceSystem(), inputBundle.GetTypingConfig(), rv.namespaceCollection, controlAttributes, @@ -723,55 +724,9 @@ func NewHandlerCtx( return &rv, nil } -func transformOpenapiStackqlAuthToLocal(authDTO formulation.AuthDTO) *dto.AuthCtx { - rv := &dto.AuthCtx{ - Scopes: authDTO.GetScopes(), - Subject: authDTO.GetSubject(), - Type: authDTO.GetType(), - ValuePrefix: authDTO.GetValuePrefix(), - KeyID: authDTO.GetKeyID(), - KeyIDEnvVar: authDTO.GetKeyIDEnvVar(), - KeyFilePath: authDTO.GetKeyFilePath(), - KeyFilePathEnvVar: authDTO.GetKeyFilePathEnvVar(), - KeyEnvVar: authDTO.GetKeyEnvVar(), - EnvVarAPIKeyStr: authDTO.GetEnvVarAPIKeyStr(), - EnvVarAPISecretStr: authDTO.GetEnvVarAPISecretStr(), - EnvVarUsername: authDTO.GetEnvVarUsername(), - EnvVarPassword: authDTO.GetEnvVarPassword(), - EncodedBasicCredentials: authDTO.GetInlineBasicCredentials(), - Location: authDTO.GetLocation(), - Name: authDTO.GetName(), - TokenURL: authDTO.GetTokenURL(), - GrantType: authDTO.GetGrantType(), - ClientID: authDTO.GetClientID(), - ClientSecret: authDTO.GetClientSecret(), - ClientIDEnvVar: authDTO.GetClientIDEnvVar(), - ClientSecretEnvVar: authDTO.GetClientSecretEnvVar(), - Values: authDTO.GetValues(), - AuthStyle: authDTO.GetAuthStyle(), - AccountID: authDTO.GetAccountID(), - AccoountIDEnvVar: authDTO.GetAccountIDEnvVar(), - // Doc-level OCI env var indirections (issue #710); a runtime --auth context - // still wins via updateAuthContextIfNotExists. - OciTenancyOCIDEnvVar: authDTO.GetOciTenancyOCIDEnvVar(), - OciUserOCIDEnvVar: authDTO.GetOciUserOCIDEnvVar(), - OciFingerprintEnvVar: authDTO.GetOciFingerprintEnvVar(), - OciPrivateKeyEnvVar: authDTO.GetOciPrivateKeyEnvVar(), - OciPrivateKeyFilePathEnvVar: authDTO.GetOciPrivateKeyPathEnvVar(), - OciPassphraseEnvVar: authDTO.GetOciPassphraseEnvVar(), - OciRegionEnvVar: authDTO.GetOciRegionEnvVar(), - } - successor, successorExists := authDTO.GetSuccessor() - currentParent := rv - for { - if successorExists { - transformedSuccessor := transformOpenapiStackqlAuthToLocal(successor) - currentParent.Successor = transformedSuccessor - currentParent = transformedSuccessor - successor, successorExists = successor.GetSuccessor() - } else { - break - } +func transformOpenapiStackqlAuthToLocal(auth *omnisdk.ProviderAuth) *dto.AuthCtx { + if auth == nil { + return nil } - return rv + return &dto.AuthCtx{AuthConfig: auth.AuthConfig, Successor: transformOpenapiStackqlAuthToLocal(auth.Successor)} } diff --git a/internal/stackql/handler/handler_test.go b/internal/stackql/handler/handler_test.go deleted file mode 100644 index b43c45f5d..000000000 --- a/internal/stackql/handler/handler_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package handler_test - -import ( - "context" - "io" - "net" - "net/http" - "net/http/httptest" - "os" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stretchr/testify/assert" -) - -func TestAwsS3BucketAclsGet(t *testing.T) { - - vr := "v0.1.0" - pb, err := os.ReadFile("./testdata/registry/src/aws/" + vr + "/provider.yaml") - if err != nil { - t.Fatalf("Test failed: could not read provider doc, error: %v", err) - } - prov, provErr := formulation.LoadProviderDocFromBytes(pb) - if provErr != nil { - t.Fatalf("Test failed: could not load provider doc, error: %v", provErr) - } - svc, err := formulation.LoadProviderAndServiceFromPaths( - "./testdata/registry/src/aws/"+vr+"/provider.yaml", - "./testdata/registry/src/aws/"+vr+"/services/s3.yaml", - ) - if err != nil { - t.Fatalf("Test failed: %v", err) - } - rsc, rscErr := svc.GetResource("bucket_acls") - if rscErr != nil { - t.Fatalf("Test failed: could not locate resource bucket_acls, error: %v", rscErr) - } - method, methodErr := rsc.FindMethod("get_bucket_acl") - if methodErr != nil { - t.Fatalf("Test failed: could not locate method get_bucket_acl, error: %v", methodErr) - } - - assert.Equal(t, svc.GetName(), "s3") - - expectedHost := "my-test-bucket.s3-ap-southeast-2.amazonaws.com" - - tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - io.Copy(io.Discard, r.Body) - assert.Equal(t, r.Host, expectedHost, "expected host does not match actual host") - w.WriteHeader(http.StatusOK) - })) - t.Cleanup(tlsServer.Close) - - baseTransport := tlsServer.Client().Transport.(*http.Transport) - dummyClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: baseTransport.TLSClientConfig, - DisableKeepAlives: true, - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return net.Dial("tcp", tlsServer.Listener.Addr().String()) - }, - }, - } - - authCtx := dto.GetAuthCtx([]string{}, "./testdata/dummy_credentials/dummy-sa-key.json", "null_auth") - - configurator := formulation.NewAnySdkClientConfigurator( - dto.RuntimeCtx{ - AllowInsecure: true, - }, - "aws", - dummyClient, - ) - httpPreparator := formulation.NewHTTPPreparator( - prov, - svc, - method, - map[int]map[string]interface{}{ - 0: { - "Bucket": "my-test-bucket", - "created_date": "2024-01-01T00:00:00Z", - "region": "ap-southeast-2", - }, - }, - nil, - nil, - logrus.StandardLogger(), - ) - armoury, armouryErr := httpPreparator.BuildHTTPRequestCtx(formulation.NewHTTPPreparatorConfig(false)) - if armouryErr != nil { - t.Fatalf("Test failed: could not build HTTP preparator armoury, error: %v", armouryErr) - } - reqParams := armoury.GetRequestParams() - if len(reqParams) < 1 { - t.Fatalf("Test failed: no request parameters found") - } - - for _, v := range reqParams { - - argList := v.GetArgList() - - response, apiErr := formulation.CallFromSignature( - configurator, - dto.RuntimeCtx{ - AllowInsecure: true, - }, - authCtx, - authCtx.Type, - false, - nil, - prov, - formulation.NewAnySdkOpStoreDesignation(method), - argList, // TODO: abstract - ) - if apiErr != nil { - t.Fatalf("Test failed: API call error: %v", apiErr) - } - if response.IsErroneous() { - t.Fatalf("Test failed: API call returned erroneous response") - } - t.Logf("Test passed: received response: %+v", response) - } - -} diff --git a/internal/stackql/handler/path_searcher.go b/internal/stackql/handler/path_searcher.go index 85e6e2711..0531287d1 100644 --- a/internal/stackql/handler/path_searcher.go +++ b/internal/stackql/handler/path_searcher.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/jsonpath" + "github.com/stackql/stackql/internal/stackql/jsonpath" ) type simpleSystemPathSearcher struct { diff --git a/internal/stackql/internal_data_transfer/builder_input/builder_input.go b/internal/stackql/internal_data_transfer/builder_input/builder_input.go index e76ef5ff6..d9cb23f8a 100644 --- a/internal/stackql/internal_data_transfer/builder_input/builder_input.go +++ b/internal/stackql/internal_data_transfer/builder_input/builder_input.go @@ -1,8 +1,6 @@ package builder_input //nolint:revive,stylecheck // permissable deviation from norm import ( - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" "github.com/stackql/stackql/internal/stackql/drm" @@ -10,6 +8,7 @@ import ( "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/provider" + "github.com/stackql/stackql/internal/stackql/streaming" "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" "github.com/stackql/stackql/internal/stackql/tablemetadata" ) @@ -29,8 +28,6 @@ type BuilderInput interface { GetParserNode() (sqlparser.SQLNode, bool) GetProvider() (provider.IProvider, bool) SetProvider(provider.IProvider) - GetOperationStore() (formulation.OperationStore, bool) - SetOperationStore(op formulation.OperationStore) IsAwait() bool IsReturning() bool SetIsReturning(bool) @@ -43,8 +40,6 @@ type BuilderInput interface { SetIsUndo(isUndo bool) SetRequiredDataRequestKey(key string) GetRequiredDataRequestKey() (string, bool) - SetPushdownLimit(limit int) - GetPushdownLimit() (int, bool) SetDependencyNode(dependencyNode primitivegraph.PrimitiveNode) SetParserNode(node sqlparser.SQLNode) SetParamMap(paramMap map[int]map[string]interface{}) @@ -53,8 +48,6 @@ type BuilderInput interface { SetParamMapStream(streaming.MapStream) SetVerb(verb string) Clone() BuilderInput - GetHTTPPreparatorStream() (formulation.HttpPreparatorStream, bool) - SetHTTPPreparatorStream(prepStream formulation.HttpPreparatorStream) IsTargetPhysicalTable() bool SetIsTargetPhysicalTable(isPhysical bool) SetTxnCtrlCtrs(internaldto.TxnControlCounters) @@ -79,8 +72,6 @@ type builderInput struct { isUndo bool node sqlparser.SQLNode paramMapStream streaming.MapStream - httpPrepStream formulation.HttpPreparatorStream - op formulation.OperationStore prov provider.IProvider annotatedAst annotatedast.AnnotatedAst isTargetPhysical bool @@ -88,8 +79,6 @@ type builderInput struct { tableInsertionContainer tableinsertioncontainer.TableInsertionContainer insertCtx drm.PreparedStatementCtx requiredDataRequestKey string - pushdownLimit int - pushdownLimitSet bool } func NewBuilderInput( @@ -110,17 +99,6 @@ func (bi *builderInput) GetRequiredDataRequestKey() (string, bool) { return bi.requiredDataRequestKey, bi.requiredDataRequestKey != "" } -// SetPushdownLimit records the LIMIT the analysis phase resolved as pushable (used by the -// GraphQL acquire path to bound the page size); the acquire only carries it out. -func (bi *builderInput) SetPushdownLimit(limit int) { - bi.pushdownLimit = limit - bi.pushdownLimitSet = true -} - -func (bi *builderInput) GetPushdownLimit() (int, bool) { - return bi.pushdownLimit, bi.pushdownLimitSet -} - func (bi *builderInput) SetRequiredDataRequestKey(key string) { bi.requiredDataRequestKey = key } @@ -184,26 +162,10 @@ func (bi *builderInput) SetProvider(prov provider.IProvider) { bi.prov = prov } -func (bi *builderInput) GetOperationStore() (formulation.OperationStore, bool) { - return bi.op, bi.op != nil -} - -func (bi *builderInput) SetOperationStore(op formulation.OperationStore) { - bi.op = op -} - func (bi *builderInput) GetParamMapStream() (streaming.MapStream, bool) { return bi.paramMapStream, bi.paramMapStream != nil } -func (bi *builderInput) GetHTTPPreparatorStream() (formulation.HttpPreparatorStream, bool) { - return bi.httpPrepStream, bi.httpPrepStream != nil -} - -func (bi *builderInput) SetHTTPPreparatorStream(prepStream formulation.HttpPreparatorStream) { - bi.httpPrepStream = prepStream -} - func (bi *builderInput) SetParamMapStream(s streaming.MapStream) { bi.paramMapStream = s } @@ -315,7 +277,5 @@ func (bi *builderInput) Clone() BuilderInput { isReturning: bi.isReturning, insertCtx: bi.insertCtx, requiredDataRequestKey: bi.requiredDataRequestKey, - pushdownLimit: bi.pushdownLimit, - pushdownLimitSet: bi.pushdownLimitSet, } } diff --git a/internal/stackql/internal_data_transfer/internaldto/db_table.go b/internal/stackql/internal_data_transfer/internaldto/db_table.go index d95839118..2e61e46ca 100644 --- a/internal/stackql/internal_data_transfer/internaldto/db_table.go +++ b/internal/stackql/internal_data_transfer/internaldto/db_table.go @@ -1,7 +1,7 @@ package internaldto import ( - "github.com/stackql/any-sdk/pkg/constants" + "github.com/stackql/stackql/internal/stackql/constants" ) var ( diff --git a/internal/stackql/internal_data_transfer/internaldto/executor_output.go b/internal/stackql/internal_data_transfer/internaldto/executor_output.go index ff1ffdf4e..cf8d15cd6 100644 --- a/internal/stackql/internal_data_transfer/internaldto/executor_output.go +++ b/internal/stackql/internal_data_transfer/internaldto/executor_output.go @@ -4,9 +4,9 @@ import ( "fmt" "sync" - "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/acid/binlog" + "github.com/stackql/stackql/internal/stackql/streaming" ) var ( diff --git a/internal/stackql/internal_data_transfer/internaldto/heirarchy.go b/internal/stackql/internal_data_transfer/internaldto/heirarchy.go index a35a0855d..625e8cfe4 100644 --- a/internal/stackql/internal_data_transfer/internaldto/heirarchy.go +++ b/internal/stackql/internal_data_transfer/internaldto/heirarchy.go @@ -4,76 +4,8 @@ import ( "fmt" "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/public/formulation" -) - -var ( - _ Heirarchy = &standardHeirarchy{} ) -type Heirarchy interface { - GetServiceHdl() formulation.Service - GetResource() formulation.Resource - GetMethodSet() formulation.MethodSet - GetMethod() formulation.StandardOperationStore - SetServiceHdl(formulation.Service) - SetResource(formulation.Resource) - SetMethodSet(formulation.MethodSet) - SetMethod(formulation.StandardOperationStore) - SetMethodStr(string) -} - -func NewHeirarchy(hIDs HeirarchyIdentifiers) Heirarchy { - return &standardHeirarchy{ - hIDs: hIDs, - } -} - -type standardHeirarchy struct { - hIDs HeirarchyIdentifiers - serviceHdl formulation.Service - resource formulation.Resource - methodSet formulation.MethodSet - method formulation.StandardOperationStore -} - -func (hr *standardHeirarchy) SetServiceHdl(sh formulation.Service) { - hr.serviceHdl = sh -} - -func (hr *standardHeirarchy) SetResource(r formulation.Resource) { - hr.resource = r -} - -func (hr *standardHeirarchy) SetMethodSet(mSet formulation.MethodSet) { - hr.methodSet = mSet -} - -func (hr *standardHeirarchy) SetMethodStr(mStr string) { - hr.hIDs.SetMethodStr(mStr) -} - -func (hr *standardHeirarchy) SetMethod(ost formulation.StandardOperationStore) { - hr.method = ost -} - -func (hr *standardHeirarchy) GetServiceHdl() formulation.Service { - return hr.serviceHdl -} - -func (hr *standardHeirarchy) GetResource() formulation.Resource { - return hr.resource -} - -func (hr *standardHeirarchy) GetMethodSet() formulation.MethodSet { - return hr.methodSet -} - -func (hr *standardHeirarchy) GetMethod() formulation.StandardOperationStore { - return hr.method -} - func generatePutativelyUniqueTableName(node sqlparser.TableName) string { if node.IsEmpty() { return "" diff --git a/internal/stackql/internal_data_transfer/internaldto/internaldto_test.go b/internal/stackql/internal_data_transfer/internaldto/internaldto_test.go index 6187f7321..5ec95309c 100644 --- a/internal/stackql/internal_data_transfer/internaldto/internaldto_test.go +++ b/internal/stackql/internal_data_transfer/internaldto/internaldto_test.go @@ -6,8 +6,8 @@ import ( "io" "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) func TestNewBasicPrimitiveContext(t *testing.T) { diff --git a/internal/stackql/internal_data_transfer/internaldto/output_ctx.go b/internal/stackql/internal_data_transfer/internaldto/output_ctx.go index 2491f6d83..c820ddb52 100644 --- a/internal/stackql/internal_data_transfer/internaldto/output_ctx.go +++ b/internal/stackql/internal_data_transfer/internaldto/output_ctx.go @@ -3,8 +3,8 @@ package internaldto import ( "time" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type OutputContext struct { diff --git a/internal/stackql/internal_data_transfer/internaldto/primitive_ctx.go b/internal/stackql/internal_data_transfer/internaldto/primitive_ctx.go index 8c98debbe..87db40ed4 100644 --- a/internal/stackql/internal_data_transfer/internaldto/primitive_ctx.go +++ b/internal/stackql/internal_data_transfer/internaldto/primitive_ctx.go @@ -3,7 +3,7 @@ package internaldto import ( "io" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) var ( diff --git a/internal/stackql/internal_data_transfer/internaldto/table_parameter_collection.go b/internal/stackql/internal_data_transfer/internaldto/table_parameter_collection.go deleted file mode 100644 index 78ed91744..000000000 --- a/internal/stackql/internal_data_transfer/internaldto/table_parameter_collection.go +++ /dev/null @@ -1,42 +0,0 @@ -package internaldto - -import "github.com/stackql/stackql/internal/stackql/suffix" - -var ( - _ TableParameterCollection = &standardTableParameterCollection{} -) - -type TableParameterCollection interface { - GetOptionalParams() suffix.ParameterSuffixMap - GetRemainingRequiredParams() suffix.ParameterSuffixMap - GetRequiredParams() suffix.ParameterSuffixMap -} - -func NewTableParameterCollection( - requiredParams, optionalParams, - remainingRequiredParameters suffix.ParameterSuffixMap, -) TableParameterCollection { - return &standardTableParameterCollection{ - requiredParams: requiredParams, - optionalParams: optionalParams, - remainingRequiredParameters: remainingRequiredParameters, - } -} - -type standardTableParameterCollection struct { - requiredParams suffix.ParameterSuffixMap - optionalParams suffix.ParameterSuffixMap - remainingRequiredParameters suffix.ParameterSuffixMap -} - -func (pc *standardTableParameterCollection) GetRequiredParams() suffix.ParameterSuffixMap { - return pc.requiredParams -} - -func (pc *standardTableParameterCollection) GetRemainingRequiredParams() suffix.ParameterSuffixMap { - return pc.remainingRequiredParameters -} - -func (pc *standardTableParameterCollection) GetOptionalParams() suffix.ParameterSuffixMap { - return pc.optionalParams -} diff --git a/internal/stackql/intrinsic/auth.go b/internal/stackql/intrinsic/auth.go new file mode 100644 index 000000000..c0bf3c087 --- /dev/null +++ b/internal/stackql/intrinsic/auth.go @@ -0,0 +1,62 @@ +package intrinsic + +import ( + "context" + "strings" + "time" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" +) + +// authFunc translates the SQL command; credential validation belongs to Omni. +func authFunc(ctx queryContext, node *sqlparser.Auth) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + configured, err := ctx.GetAuthContext(node.Provider) + if err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + candidate := configured.Clone() + if node.KeyFilePath != "" { + candidate.KeyFilePath = node.KeyFilePath + } + if node.KeyEnvVar != "" { + candidate.KeyEnvVar = node.KeyEnvVar + } + candidate.Type = strings.ToLower(node.Type) + if candidate.Type == "" { + candidate.Type = dto.AuthNullStr + if candidate.KeyFilePath != "" || candidate.KeyEnvVar != "" { + candidate.Type = dto.AuthServiceAccountStr + } + } + credential := omnisdk.NewCredential(node.Provider, *dto.OmniAuth(candidate)) + if err = credential.Check(); err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + candidate.Active = true + if principal, principalErr := credential.Principal(); principalErr == nil { + candidate.ID = principal + } + *configured = *candidate + return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) + } +} + +func authRevokeFunc(ctx queryContext, node *sqlparser.AuthRevoke) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + configured, err := ctx.GetAuthContext(node.Provider) + if err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + deadline, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + err = omnisdk.NewCredential(node.Provider, *dto.OmniAuth(configured)).Revoke(deadline) + if err == nil { + configured.Active, configured.ID = false, "" + } + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } +} diff --git a/internal/stackql/intrinsic/dialect.go b/internal/stackql/intrinsic/dialect.go index c3c1fbf39..c67a22984 100644 --- a/internal/stackql/intrinsic/dialect.go +++ b/internal/stackql/intrinsic/dialect.go @@ -5,8 +5,8 @@ import ( "github.com/stackql-labs/omnisdk/pkg/query" "github.com/stackql-labs/omnisdk/pkg/sqlfn" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) // sqlDialect is the SQL of the session's backend as the translator writes it: the omnisdk function diff --git a/internal/stackql/intrinsic/dialect_test.go b/internal/stackql/intrinsic/dialect_test.go index 4d00a6238..1db387d1a 100644 --- a/internal/stackql/intrinsic/dialect_test.go +++ b/internal/stackql/intrinsic/dialect_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stackql-labs/omnisdk/pkg/sqlfn" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) // backendCtx is a queryContext whose only state is the SQL backend's configuration. diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go index cfc562791..7929975ef 100644 --- a/internal/stackql/intrinsic/doc.go +++ b/internal/stackql/intrinsic/doc.go @@ -4,17 +4,17 @@ import ( "context" "fmt" "path/filepath" + "slices" "sort" "strings" "github.com/stackql-labs/omnisdk/pkg/docparse/aot" "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql-parser/go/vt/sqlparser" @@ -26,37 +26,23 @@ import ( // its constant rather than a copy of the literal. const UnstablePrefix = aot.DefaultProviderPrefix -// IsUnstableEnabled reports whether the document-driven providers were opted -// into. They are documents read straight from disk, with none of the registry's -// curation behind them, so nothing exposes them until a caller asks. -func IsUnstableEnabled() bool { - return previewCfg.getUnstableEnabled() || previewCfg.getOmniAll() -} - -// IsExtendedCatalogEnabled keeps the compatibility catalogue stable when -// Omni replaces the execution backend. Explicit unstable mode adds SDK services. +// IsExtendedCatalogEnabled reports whether the preview catalogue was requested. func IsExtendedCatalogEnabled() bool { - return previewCfg.getUnstableEnabled() && !previewCfg.getOmniAll() + return previewCfg.getUnstableEnabled() } -// docProvider is the bundle behind an unstable provider name, or false. Once -// every provider is routed to omnisdk, an unprefixed name is one too. +// docProvider resolves provider documents through omnisdk. The built-in preview +// provider keeps its own routing, including the opt-in IaC service. func docProvider(name string) (string, bool) { - if !IsUnstableEnabled() { - return "", false - } trimmed := strings.TrimSpace(name) - if !strings.HasPrefix(strings.ToLower(trimmed), UnstablePrefix) { - if !previewCfg.getOmniAll() || trimmed == "" || strings.EqualFold(trimmed, ProviderName) { - return "", false - } - return trimmed, true - } - bundle := trimmed[len(UnstablePrefix):] - if bundle == "" { + if trimmed == "" || strings.EqualFold(trimmed, ProviderName) { return "", false } - return bundle, true + bundle := trimmed + if strings.HasPrefix(strings.ToLower(trimmed), UnstablePrefix) { + bundle = trimmed[len(UnstablePrefix):] + } + return bundle, bundle != "" } // bundleAliases maps the provider name stackql presents onto the directory the @@ -77,7 +63,7 @@ func bundleDir(label string) string { // the canonical providers resolve it, so an unstable provider reads whatever // documents are already on disk rather than needing assets of its own. func localDocRoot(runtimeCtx dto.RuntimeCtx) string { - var registryCfg formulation.RegistryConfig + var registryCfg omnisdk.RegistryConfig if err := yaml.Unmarshal([]byte(runtimeCtx.RegistryRaw), ®istryCfg); err == nil { if registryCfg.LocalDocRoot != "" { return registryCfg.LocalDocRoot @@ -97,7 +83,7 @@ func localDocRoot(runtimeCtx dto.RuntimeCtx) string { func docRoot(ctx queryContext, bundle string) (string, error) { runtime := ctx.GetRuntimeContext() prefix := "src" - var registryCfg formulation.RegistryConfig + var registryCfg omnisdk.RegistryConfig if err := yaml.Unmarshal([]byte(runtime.RegistryRaw), ®istryCfg); err != nil { return "", err } @@ -151,8 +137,17 @@ func docMethods(ctx queryContext, bundle, service, resource string) ([]relationM } out := make([]relationMethod, 0, len(methods)) for _, method := range methods { + required := unresolvedServerParams(method) + if method.Name != "__view__" { + signature, describeErr := omnisdk.DescribeMethod( + registryRoot(ctx), UnstablePrefix+bundle+"."+service+"."+resource, method.Name, opts...) + if describeErr != nil { + return nil, describeErr + } + required = requiredMethodInputs(method, signature) + } out = append(out, relationMethod{ - requiredParams: unresolvedServerParams(method), + requiredParams: required, name: method.Name, description: method.OperationID, sqlVerb: docSQLVerb(method.SQLVerb), @@ -161,6 +156,56 @@ func docMethods(ctx queryContext, bundle, service, resource string) ([]relationM return out, nil } +// describeDocument obtains response metadata from the same SDK catalogue used +// for execution, without selecting or executing a provider operation in the host. +func describeDocument( + ctx queryContext, bundle, service, resource, method string, extended bool, +) internaldto.ExecutorOutput { + opts, err := documentOptions(ctx) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + address := UnstablePrefix + bundle + "." + service + "." + resource + if method == "" { + root, rootErr := docRoot(ctx, bundle) + if rootErr != nil { + return internaldto.NewErroneousExecutorOutput(rootErr) + } + view, exists, viewErr := omnisdk.DocView(root, bundle, service, resource, opts...) + if viewErr != nil { + return internaldto.NewErroneousExecutorOutput(viewErr) + } + if exists { + return describeDocumentView(ctx, view, extended) + } + } + var signature omnisdk.MethodSignature + if method != "" { + signature, err = omnisdk.DescribeMethod(registryRoot(ctx), address, method, opts...) + } else { + var declared omnisdk.Table + declared, err = omnisdk.DescribeTable(registryRoot(ctx), address, opts...) + if err == nil && len(declared.Methods()) > 0 { + signature = declared.Methods()[0] + } + } + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + if signature == nil { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("%s declares no response schema", address)) + } + types := signature.ColumnTypes() + columns := make([]column, 0, len(signature.Columns())) + for _, name := range signature.Columns() { + columns = append(columns, column{name: name, dataType: types[name].Type}) + } + if method != "" { + return describeDocumentMethod(ctx, signature, columns, extended) + } + return describeDocumentColumns(ctx, columns, extended) +} + // docSQLVerb is the verb a document maps a method to, upper-cased as SHOW // METHODS reports it. A method no verb maps is reachable only through EXEC. func docSQLVerb(verb string) string { @@ -188,7 +233,7 @@ func docSelectFunc( if relationErr != nil { return refuse(relationErr), true } - if previewCfg.getStagingEnabled() && needsStaging(node) { + if needsStaging(node) { return stagedSelectWithRelations(ctx, node, currentProvider, relations), true } dialect, err := backendDialect(ctx) @@ -251,8 +296,17 @@ func openDocQuery(ctx queryContext, translated docQuery) (omnisdk.Rows, string, return openDocQueryIn(ctx, translated, context.Background()) } -//nolint:gocognit,funlen // The cognitive complexity of this function is high but acceptable for now func openDocQueryIn(ctx queryContext, translated docQuery, runCtx context.Context) (omnisdk.Rows, string, error) { + plan, relation, err := planDocQuery(ctx, translated) + if err != nil { + return nil, "", err + } + rows, err := plan.Open(runCtx) + return rows, relation, err +} + +//nolint:gocognit,funlen // Resolving metadata and assembling a plan never executes it. +func planDocQuery(ctx queryContext, translated docQuery) (omnisdk.Plan, string, error) { opts, err := documentOptions(ctx) if err != nil { return nil, "", err @@ -292,7 +346,11 @@ func openDocQueryIn(ctx queryContext, translated docQuery, runCtx context.Contex if dialectErr != nil { return nil, "", dialectErr } - q, environmentErr := bindServerEnvironment(q, tables) + q, environmentErr := bindReadPageSize(q, tables, ctx.GetRuntimeContext().HTTPMaxResults) + if environmentErr != nil { + return nil, "", environmentErr + } + q, environmentErr = bindServerEnvironment(q, tables) if environmentErr != nil { return nil, "", environmentErr } @@ -322,11 +380,7 @@ func openDocQueryIn(ctx queryContext, translated docQuery, runCtx context.Contex if planErr != nil { return nil, "", planErr } - rows, openErr := plan.Open(runCtx) - if openErr != nil { - return nil, "", openErr - } - return rows, relation, nil + return plan, relation, nil } // queryProviderAuth preserves configured credentials independently for every provider. @@ -418,13 +472,18 @@ func declaredEmptyColumns(ctx queryContext, translated docQuery) []column { // effects, and reports the outcome. func drainMutation(rows omnisdk.Rows) internaldto.ExecutorOutput { defer rows.Close() - for rows.Next() { //nolint:revive // draining sends the effects + var messages []string + for rows.Next() { + messages = append(messages, mutationSuccessMessage) } if err := rows.Err(); err != nil { return internaldto.NewErroneousExecutorOutput(err) } + if len(messages) == 0 { + messages = append(messages, mutationSuccessMessage) + } return internaldto.NewExecutorOutput(nil, nil, nil, - internaldto.NewBackendMessages([]string{mutationSuccessMessage}), nil) + internaldto.NewBackendMessages(messages), nil) } func refuse(err error) func() internaldto.ExecutorOutput { @@ -461,7 +520,7 @@ func showDocServices( } rows[fmt.Sprintf("%06d", i)] = row } - return prepareDocMetadata(ctx, formulation.GetServicesHeader(extended), rows, filters) + return prepareDocMetadata(ctx, util.GetServicesHeader(extended), rows, filters) } func showDocResources( @@ -477,7 +536,7 @@ func showDocResources( rows := make(map[string]map[string]interface{}, len(tables)) for i, tbl := range tables { displayName := tbl.name - if declared, present := names[tbl.name]; present { + if declared, present := names[tbl.name]; present && declared != "" { displayName = declared } row := map[string]interface{}{ @@ -489,7 +548,7 @@ func showDocResources( } rows[fmt.Sprintf("%06d", i)] = row } - return prepareDocMetadata(ctx, formulation.GetResourcesHeader(extended), rows, filters) + return prepareDocMetadata(ctx, util.GetResourcesHeader(extended), rows, filters) } func showDocMethods( @@ -542,3 +601,31 @@ func configureReadHints(translated docQuery, args *omnisdk.Args) { args.ReadHintsByAlias = hinted.getReadHints() } } + +func requiredMethodInputs(method omnisdk.DocMethod, signature omnisdk.MethodSignature) []string { + required := unresolvedServerParams(method) + metadata, ok := signature.(omnisdk.RequestMetadata) + if !ok || metadata.RequestBodySchema() == nil { + return required + } + body, declared := metadata.RequestBodySchema().(aot.RequiredSchema) + if !declared { + return required + } + for _, name := range body.RequiredProperties() { + input := "data__" + metadata.InputName(name) + if !slices.Contains(required, input) { + required = append(required, input) + } + } + return required +} + +func describeDocumentColumns(ctx queryContext, columns []column, extended bool) internaldto.ExecutorOutput { + rows := columnRows(columns, extended, nil) + named := make(map[string]map[string]interface{}, len(rows)) + for i, column := range columns { + named[column.name] = rows[fmt.Sprintf("%06d", i)] + } + return prepare(ctx, util.GetDescribeHeader(extended), named, util.DescribeRowSort) +} diff --git a/internal/stackql/intrinsic/doc_metadata.go b/internal/stackql/intrinsic/doc_metadata.go index 7e79e56b3..a44e695f7 100644 --- a/internal/stackql/intrinsic/doc_metadata.go +++ b/internal/stackql/intrinsic/doc_metadata.go @@ -7,6 +7,7 @@ import ( "sort" "strings" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/util" @@ -61,7 +62,11 @@ func prepareDocMetadata( fields := make([]string, 0, len(names)) args := make([]any, 0, len(names)) for i, name := range names { - fields = append(fields, fmt.Sprintf("$%d AS \"%s\"", i+1, strings.ReplaceAll(name, "\"", "\"\""))) + parameterType := "TEXT" + if _, boolean := row[name].(bool); boolean { + parameterType = "BOOLEAN" + } + fields = append(fields, fmt.Sprintf("CAST($%d AS %s) AS %s", i+1, parameterType, quoteRelationIdentifier(name))) args = append(args, row[name]) } statement := "SELECT 1 FROM (SELECT " + strings.Join(fields, ",") + ") AS metadata WHERE " + formatter.String() @@ -75,3 +80,39 @@ func prepareDocMetadata( } return prepare(ctx, columns, rows, util.DefaultRowSort) } + +func describeDocumentMethod( + ctx queryContext, signature omnisdk.MethodSignature, columns []column, extended bool, +) internaldto.ExecutorOutput { + order := []string{"name", "type", "param_type", "shape"} + if extended { + order = append(order, "description") + } + rows := make(map[string]map[string]interface{}) + appendRow := func(name, kind, parameterType string) { + row := map[string]interface{}{"name": name, "type": kind, "param_type": parameterType, "shape": kind} + if extended { + row["description"] = "" + } + rows[fmt.Sprintf("%06d", len(rows))] = row + } + for _, parameter := range signature.Params() { + name := parameter.Name() + if parameter.In() == "body" { + name = "data__" + name + } + kind := "string" + if typed, ok := parameter.(interface{ Type() string }); ok && typed.Type() != "" { + kind = typed.Type() + } + parameterType := "input_optional" + if parameter.Required() { + parameterType = "input_required" + } + appendRow(name, kind, parameterType) + } + for _, col := range columns { + appendRow(col.name, col.dataType, "output") + } + return prepare(ctx, order, rows, util.DefaultRowSort) +} diff --git a/internal/stackql/intrinsic/doc_views.go b/internal/stackql/intrinsic/doc_views.go index fd3fddee1..04afc7797 100644 --- a/internal/stackql/intrinsic/doc_views.go +++ b/internal/stackql/intrinsic/doc_views.go @@ -8,6 +8,7 @@ import ( "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" ) // expandDocumentViews substitutes a single FROM view before the normal SDK @@ -84,7 +85,7 @@ func expandDocumentViews( return nil, fmt.Errorf("view %s requires a separate relational staging boundary", key) } // Clone the caller before replacing any AST references. - clone, err := sqlparser.Parse(sqlparser.String(node)) + clone, err := sqlparser.Parse(statementSQL(node)) if err != nil { return nil, err } @@ -229,3 +230,32 @@ func inlineDocumentView(node, view *sqlparser.Select, alias string) (*sqlparser. node.From = view.From return node, nil } + +// View SQL belongs to the host; describe its declared projections without executing it. +func describeDocumentView(ctx queryContext, view aot.View, extended bool) internaldto.ExecutorOutput { + statement, err := sqlparser.Parse(view.DDL()) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + node, ok := statement.(*sqlparser.Select) + if !ok { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("view must declare SELECT")) + } + columns := make([]column, 0, len(node.SelectExprs)) + for _, expression := range node.SelectExprs { + selection, isSelection := expression.(*sqlparser.AliasedExpr) + if !isSelection { + continue + } + name, nameErr := projectionName(selection) + if nameErr != nil { + return internaldto.NewErroneousExecutorOutput(nameErr) + } + typ := "text" + if _, bare := selection.Expr.(*sqlparser.ColName); bare { + typ = "unknown" + } + columns = append(columns, column{name: name, dataType: typ}) + } + return describeDocumentColumns(ctx, columns, extended) +} diff --git a/internal/stackql/intrinsic/exec.go b/internal/stackql/intrinsic/exec.go new file mode 100644 index 000000000..ecf33240c --- /dev/null +++ b/internal/stackql/intrinsic/exec.go @@ -0,0 +1,199 @@ +package intrinsic + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" +) + +// Legacy EXEC names a method explicitly; the SDK still validates and executes it. +func execPlan(ctx queryContext, node *sqlparser.Exec) (omnisdk.Plan, omnisdk.MethodSignature, error) { + name := node.MethodName + provider := resolveProvider(name.QualifierThird.GetRawVal(), ctx.GetCurrentProvider()) + bundle, ok := docProvider(provider) + if !ok { + return nil, nil, fmt.Errorf("EXEC requires a document provider") + } + address := UnstablePrefix + bundle + "." + name.QualifierSecond.GetRawVal() + "." + name.Qualifier.GetRawVal() + method := name.Name.GetRawVal() + opts, err := documentOptions(ctx) + if err != nil { + return nil, nil, err + } + signature, err := omnisdk.DescribeMethod(registryRoot(ctx), address, method, opts...) + if err != nil { + return nil, nil, err + } + params := map[string]string{} + for _, variable := range node.ExecVarDefs { + value, valueErr := execLiteral(variable.Val) + if valueErr != nil { + return nil, nil, valueErr + } + name := execInputName(signature, variable.ColIdent.GetRawVal()) + params[name] = fmt.Sprint(textValue(value)) + } + body, err := execPayload(node.OptExecPayload, signature) + if err != nil { + return nil, nil, err + } + args, err := previewArgs(ctx, bundle, nil) + if err != nil { + return nil, nil, err + } + if node.Await || sqlparser.ExtractCommentDirectives(node.Comments).IsSet("AWAIT") { + args.MutationCompletion, err = newOperationCompletion(ctx, address, false) + if err != nil { + return nil, nil, err + } + } + graph, err := omnisdk.NewGraph([]omnisdk.Node{omnisdk.NewExecNode("exec", address, method, params, body)}, nil) + if err != nil { + return nil, nil, err + } + plan, err := omnisdk.NewGraphQuery(registryRoot(ctx), graph, args) + return plan, signature, err +} + +func execPayload(payload *sqlparser.ExecVarDef, signature omnisdk.MethodSignature) (map[string]any, error) { + if payload == nil { + return nil, nil //nolint:nilnil // No payload was supplied. + } + if !signature.TakesBody() { + return nil, fmt.Errorf("method %s has no request body schema", signature.Method()) + } + if !strings.EqualFold(payload.ColIdent.GetRawVal(), "json") { + return nil, fmt.Errorf("EXEC supports only a JSON payload") + } + value, err := execLiteral(payload.Val) + if err != nil { + return nil, err + } + var body map[string]any + if err = json.Unmarshal([]byte(fmt.Sprint(value)), &body); err != nil { + return nil, err + } + return body, nil +} + +func execLiteral(expr sqlparser.Expr) (any, error) { + value, err := expression(expr) + if err != nil { + return nil, err + } + literal, ok := value.(query.Literal) + if !ok { + return nil, fmt.Errorf("EXEC arguments must be literal values") + } + return literal.Value(), nil +} + +func execFunc(ctx queryContext, node *sqlparser.Exec) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + plan, _, err := execPlan(ctx, node) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows, err := plan.Open(context.Background()) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + result := drainMutation(rows) + if result.GetError() == nil && (node.Await || sqlparser.ExtractCommentDirectives(node.Comments).IsSet("AWAIT")) { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) + } + return result + } +} + +// An EXEC used as a read source is a statement-local SDK relation. +func prepareExecRelations( + ctx queryContext, original *sqlparser.Select, +) (*sqlparser.Select, map[string]omnisdk.Relation, error) { + clone := *original + relations := map[string]omnisdk.Relation{} + var replace func(sqlparser.TableExpr) (sqlparser.TableExpr, error) + replace = func(expr sqlparser.TableExpr) (sqlparser.TableExpr, error) { + switch node := expr.(type) { + case *sqlparser.ExecSubquery: + return replace(&sqlparser.AliasedTableExpr{Expr: node}) + case *sqlparser.JoinTableExpr: + cloned := *node + var err error + cloned.LeftExpr, err = replace(node.LeftExpr) + if err != nil { + return nil, err + } + cloned.RightExpr, err = replace(node.RightExpr) + return &cloned, err + case *sqlparser.AliasedTableExpr: + return execRelation(ctx, node, relations) + } + return expr, nil + } + clone.From = make(sqlparser.TableExprs, len(original.From)) + for i, table := range original.From { + var err error + clone.From[i], err = replace(table) + if err != nil { + return nil, nil, err + } + } + return &clone, relations, nil +} + +func execRelation( + ctx queryContext, node *sqlparser.AliasedTableExpr, relations map[string]omnisdk.Relation, +) (sqlparser.TableExpr, error) { + execution, ok := node.Expr.(*sqlparser.ExecSubquery) + if !ok { + return node, nil + } + plan, signature, err := execPlan(ctx, execution.Exec) + if err != nil { + return nil, err + } + local := fmt.Sprintf("exec%d", len(relations)) + handle := UnstablePrefix + "omni_relations.sources." + local + columns := signature.ColumnTypes() + if columns == nil { + columns = map[string]omnisdk.ColumnType{} + } + for _, name := range signature.Columns() { + if _, present := columns[name]; !present { + columns[name] = omnisdk.ColumnType{} + } + } + relation, err := omnisdk.NewRelation(handle, nil, columns, + func(runCtx context.Context, _ map[string]any) (omnisdk.Rows, error) { + return plan.Open(runCtx) + }) + if err != nil { + return nil, err + } + relations[handle] = relation + cloned := *node + cloned.Expr = sqlparser.TableName{ + QualifierSecond: sqlparser.NewTableIdent(UnstablePrefix + "omni_relations"), + Qualifier: sqlparser.NewTableIdent("sources"), + Name: sqlparser.NewTableIdent(local), + } + return &cloned, nil +} + +func execInputName(signature omnisdk.MethodSignature, name string) string { + if metadata, ok := signature.(omnisdk.RequestMetadata); ok { + for _, parameter := range signature.Params() { + if metadata.InputName(parameter.Name()) == name { + return parameter.Name() + } + } + } + return name +} diff --git a/internal/stackql/intrinsic/intrinsic.go b/internal/stackql/intrinsic/intrinsic.go index 643b6001a..c66c748e6 100644 --- a/internal/stackql/intrinsic/intrinsic.go +++ b/internal/stackql/intrinsic/intrinsic.go @@ -4,10 +4,9 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/internal/stackql/util" @@ -61,6 +60,12 @@ func GeneratePrimitiveFunc( ) (func() internaldto.ExecutorOutput, bool) { current := ctx.GetCurrentProvider() switch node := stmt.(type) { + case *sqlparser.Auth: + return authFunc(ctx, node), true + case *sqlparser.AuthRevoke: + return authRevokeFunc(ctx, node), true + case *sqlparser.Sleep: + return sleepFunc(node), true case *sqlparser.Use: return useFunc(ctx, node) case *sqlparser.Show: @@ -79,6 +84,18 @@ func GenerateStreamFunc( ctx queryContext, stmt sqlparser.Statement, ) (func() internaldto.ExecutorOutput, bool) { + switch node := stmt.(type) { + case *sqlparser.Union: + return statementReadFunc(ctx, node), true + case *sqlparser.ParenSelect: + return statementReadFunc(ctx, node), true + } + if node, isExec := stmt.(*sqlparser.Exec); isExec { + return execFunc(ctx, node), true + } + if execute, handled := materializedFunc(ctx, stmt); handled { + return execute, true + } if node, isSelect := stmt.(*sqlparser.Select); isSelect { return selectFunc(ctx, node, ctx.GetCurrentProvider()) } @@ -135,7 +152,7 @@ func useFunc(ctx queryContext, node *sqlparser.Use) (func() internaldto.Executor return nil, false } return func() internaldto.ExecutorOutput { - ctx.SetCurrentProvider(ProviderName) + ctx.SetCurrentProvider(node.DBName.GetRawVal()) return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) }, true } @@ -151,6 +168,10 @@ func showFunc( return fn, true } switch strings.ToUpper(strings.TrimSpace(node.Type)) { + case "INSERT": + if bundle, isDoc := docProvider(resolveProvider(node.OnTable.QualifierSecond.GetRawVal(), currentProvider)); isDoc { + return func() internaldto.ExecutorOutput { return showDocInsert(ctx, node, bundle) }, true + } case "SERVICES": provider := resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider) if bundle, isDoc := docProvider(provider); isDoc { @@ -206,6 +227,15 @@ func describeTableFunc( if fn, isPreview := describePreviewTableFunc(ctx, node, currentProvider); isPreview { return fn, true } + if fn, stored := describeStoredFunc(ctx, node, currentProvider); stored { + return fn, true + } + if bundle, ok := docProvider(resolveProvider(node.Table.QualifierSecond.GetRawVal(), currentProvider)); ok { + return func() internaldto.ExecutorOutput { + return describeDocument(ctx, bundle, node.Table.Qualifier.GetRawVal(), + node.Table.Name.GetRawVal(), "", isExtended(node.Extended)) + }, true + } tbl, ok := lookupTable( node.Table.QualifierSecond.GetRawVal(), node.Table.Qualifier.GetRawVal(), @@ -224,6 +254,12 @@ func describeMethodFunc( node *sqlparser.DescribeMethod, currentProvider string, ) (func() internaldto.ExecutorOutput, bool) { + if bundle, ok := docProvider(resolveProvider(node.Provider.GetRawVal(), currentProvider)); ok { + return func() internaldto.ExecutorOutput { + return describeDocument(ctx, bundle, node.Service.GetRawVal(), + node.Resource.GetRawVal(), node.Method.GetRawVal(), isExtended(node.Extended)) + }, true + } tbl, ok := lookupTable( node.Provider.GetRawVal(), node.Service.GetRawVal(), @@ -275,7 +311,7 @@ func showServices(ctx queryContext, extended bool) internaldto.ExecutorOutput { } rows := map[string]map[string]interface{}{"000001": row} appendPreviewServices(rows, extended) - return prepare(ctx, formulation.GetServicesHeader(extended), rows, util.DefaultRowSort) + return prepare(ctx, util.GetServicesHeader(extended), rows, util.DefaultRowSort) } func showResources(ctx queryContext, serviceStr string, extended bool) internaldto.ExecutorOutput { @@ -300,7 +336,7 @@ func showResources(ctx queryContext, serviceStr string, extended bool) internald } rows[fmt.Sprintf("%06d", i)] = row } - return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort) + return prepare(ctx, util.GetResourcesHeader(extended), rows, util.DefaultRowSort) } func showMethods(ctx queryContext, tbl table, extended bool) internaldto.ExecutorOutput { @@ -326,7 +362,7 @@ func showMethods(ctx queryContext, tbl table, extended bool) internaldto.Executo func describeTable(ctx queryContext, tbl table, extended bool) internaldto.ExecutorOutput { return prepare( ctx, - formulation.GetDescribeHeader(extended), + util.GetDescribeHeader(extended), columnRows(tbl.columns, extended, nil), util.DescribeRowSort, ) @@ -365,3 +401,37 @@ func columnRows( } return rows } + +// ValidateStatement builds an Omni plan without opening its cursor or dispatching requests. +func ValidateStatement(ctx queryContext, stmt sqlparser.Statement) (bool, error) { + dialect, err := backendDialect(ctx) + if err != nil { + return true, err + } + translator := newDocTranslator(ctx.GetCurrentProvider(), dialect) + var translated docQuery + switch node := stmt.(type) { + case *sqlparser.Select: + handled, sourceErr := fromDocProviders(node.From, ctx.GetCurrentProvider()) + if !handled { + return false, sourceErr + } + if sourceErr != nil { + return true, sourceErr + } + _, _, err = planStatementRead(ctx, node, ctx.GetCurrentProvider(), 0, nil) + return true, err + case *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete: + translated, err = translator.mutation(stmt) + case *sqlparser.Exec: + _, _, err = execPlan(ctx, node) + return true, err + default: + return false, nil + } + if err != nil { + return true, err + } + _, _, err = planDocQuery(ctx, translated) + return true, err +} diff --git a/internal/stackql/intrinsic/legacy_read_cache.go b/internal/stackql/intrinsic/legacy_read_cache.go index 2a2ded01d..fb5833f87 100644 --- a/internal/stackql/intrinsic/legacy_read_cache.go +++ b/internal/stackql/intrinsic/legacy_read_cache.go @@ -10,14 +10,12 @@ import ( "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" "github.com/stackql-labs/omnisdk/pkg/rowcache" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/stackql/internal/stackql/provider" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/tablenamespace" ) type legacyCacheHost interface { queryContext - GetProvider(string) (provider.IProvider, error) GetNamespaceCollection() tablenamespace.Collection } @@ -55,11 +53,15 @@ func (c *legacyNamespaceCache) Lookup(ctx context.Context, key rowcache.Key) (ro if len(pieces) != qualifiedResourceParts { return nil, false, nil } - table, names, err := c.metadata(canonical, pieces, key.Method) + table, names, err := c.metadata(canonical, key.Method) if err != nil { return nil, false, err } + if table == "" { + return nil, false, nil + } + encoded, err := c.requestEncoding(canonical, key) if err != nil { return nil, false, err @@ -132,52 +134,6 @@ func configureLegacyReadCache(ctx queryContext, q query.Unresolved, args *omnisd } } -func (c *legacyNamespaceCache) metadata( - canonical string, pieces []string, methodName string, -) (string, []string, error) { - prov, err := c.host.GetProvider(pieces[0]) - if err != nil { - return "", nil, err - } - resource, err := prov.GetResource(pieces[1], pieces[2], c.host.GetRuntimeContext()) - if err != nil { - return "", nil, err - } - method, err := resource.FindMethod(methodName) - if err != nil { - return "", nil, err - } - schema, _, err := method.GetSelectSchemaAndObjectPath() - if err != nil || schema == nil { - return "", nil, err - } - selectionName := schema.GetSelectionName() - if selectionName == "" { - response, _, responseErr := method.GetResponseBodySchemaAndMediaType() - if responseErr != nil { - return "", nil, responseErr - } - selectionName = response.GetSelectionName() - } - if schema.GetType() == "array" { - schema, err = schema.GetItemsSchema() - if err != nil { - return "", nil, err - } - } - table := canonical + "." + selectionName - var names []string - properties, err := schema.GetProperties() - if err != nil { - return "", nil, err - } - for name := range properties { - names = append(names, name) - } - sort.Strings(names) - return table, names, nil -} - // Cache hits stream from the host's SQL cursor; no table-sized row slice is built. type legacyCacheRows struct { rows *sql.Rows @@ -228,3 +184,21 @@ func (r *legacyCacheRows) Next(ctx context.Context) bool { func (r *legacyCacheRows) Row() map[string]any { return r.current } func (r *legacyCacheRows) Err() error { return r.err } func (r *legacyCacheRows) Close() error { r.stop(); return r.rows.Close() } + +func (c *legacyNamespaceCache) metadata(canonical, method string) (string, []string, error) { + opts, err := documentOptions(c.host) + if err != nil { + return "", nil, err + } + signature, err := omnisdk.DescribeMethod(registryRoot(c.host), UnstablePrefix+canonical, method, opts...) + if err != nil { + return "", nil, err + } + named, ok := signature.(omnisdk.NamedResponse) + if !ok || named.RowSchemaName() == "" { + return "", nil, nil + } + names := append([]string(nil), signature.Columns()...) + sort.Strings(names) + return canonical + "." + named.RowSchemaName(), names, nil +} diff --git a/internal/stackql/intrinsic/materialized.go b/internal/stackql/intrinsic/materialized.go new file mode 100644 index 000000000..f83e21b76 --- /dev/null +++ b/internal/stackql/intrinsic/materialized.go @@ -0,0 +1,134 @@ +package intrinsic + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/parserutil" + "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/typing" +) + +func materializedFunc(ctx queryContext, stmt sqlparser.Statement) (func() internaldto.ExecutorOutput, bool) { + var selected sqlparser.SelectStatement + var name, ddl string + var replace, refresh bool + switch node := stmt.(type) { + case *sqlparser.DDL: + if !parserutil.IsCreateMaterializedView(node) { + return nil, false + } + selected, name, replace = node.SelectStatement, qualifiedName(node.Table), node.OrReplace + ddl = "create materialized view " + sqlparser.String(node.Table) + " as " + statementSQL(selected) + case *sqlparser.RefreshMaterializedView: + selected, name, refresh = node.ImplicitSelect, qualifiedName(node.ViewName), true + default: + return nil, false + } + source := selected + host, ok := ctx.(interface{ GetSQLSystem() sql_system.SQLSystem }) + if !ok { + return nil, false + } + return func() internaldto.ExecutorOutput { + err := materialize(ctx, host.GetSQLSystem(), source, name, ddl, replace, refresh) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + message := "DDL Execution Completed" + if refresh { + message = "refresh materialized view completed" + } + return internaldto.NewExecutorOutput(nil, nil, nil, + internaldto.NewBackendMessages([]string{message}), nil) + }, true +} + +// Provider acquisition belongs to Omni; the host only persists the finished relation. +func materialize( + ctx queryContext, system sql_system.SQLSystem, original sqlparser.SelectStatement, + name, ddl string, replace, refresh bool, +) error { + names, open, err := planStatementRead(ctx, original, ctx.GetCurrentProvider(), 0, nil) + if err != nil { + return err + } + bg := context.Background() + rows, err := open(bg, nil) + if err != nil { + return err + } + defer rows.Close() + manager, err := getStagingManager(ctx) + if err != nil { + return err + } + id, err := manager.Begin(bg) + if err != nil { + return err + } + defer manager.Release(bg, id) //nolint:errcheck // The statement outcome takes precedence over cleanup. + batch := newOmniBatchSource(rows, names, previewCfg.getBatchSize()) + first, err := batch.Next(bg) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + batch.pending = first + staged := stagedColumns(names, first) + if _, err = manager.Stage(bg, id, staged, batch); err != nil { + return err + } + if batch.terminalErr != nil { + return batch.terminalErr + } + columns := make([]typing.RelationalColumn, len(staged)) + for i, column := range staged { + columns[i] = typing.NewRelationalColumn(column.Name(), column.RelationalType()) + } + query := fmt.Sprintf("SELECT * FROM %s", manager.TableName(id)) + if refresh { + return system.RefreshMaterializedView(name, columns, query) + } + if _, exists := system.GetMaterializedViewByName(name); replace && exists { + if err = system.DropMaterializedView(name); err != nil { + return err + } + } + return system.CreateMaterializedView(system.GetFullyQualifiedRelationName(name), columns, ddl, replace, query) +} + +// Preserve qualification across SQL serialization; the parser's default table +// formatter quotes the entire address as one identifier. +func statementSQL(statement sqlparser.SQLNode) string { + buffer := sqlparser.NewTrackedBuffer(func(buf *sqlparser.TrackedBuffer, node sqlparser.SQLNode) { + if table, ok := node.(sqlparser.TableName); ok { + first := true + parts := []sqlparser.TableIdent{table.QualifierThird, table.QualifierSecond, table.Qualifier, table.Name} + for _, part := range parts { + if part.IsEmpty() { + continue + } + if !first { + buf.WriteString(".") + } + buf.WriteString(quoteRelationIdentifier(part.GetRawVal())) + first = false + } + return + } + switch identifier := node.(type) { + case sqlparser.ColIdent: + buf.WriteString(quoteRelationIdentifier(identifier.GetRawVal())) + case sqlparser.TableIdent: + buf.WriteString(quoteRelationIdentifier(identifier.GetRawVal())) + default: + node.Format(buf) + } + }) + buffer.Myprintf("%v", statement) + return buffer.String() +} diff --git a/internal/stackql/intrinsic/mutation_relations.go b/internal/stackql/intrinsic/mutation_relations.go index 7e0732689..8af261e38 100644 --- a/internal/stackql/intrinsic/mutation_relations.go +++ b/internal/stackql/intrinsic/mutation_relations.go @@ -25,14 +25,11 @@ func prepareMutationRelations( if !ok || literalSource(source.From) { return stmt, nil, nil } - expanded, err := expandDocumentViews(ctx, source, current) - if err != nil { - return nil, nil, err - } - prepared, relations, err := resolveDerivedRelations(ctx, expanded, current, 0) + prepared, relations, err := prepareMutationSource(ctx, source, current) if err != nil { return nil, nil, err } + dialect, err := backendDialect(ctx) if err != nil { return nil, nil, err @@ -41,6 +38,7 @@ func prepareMutationRelations( if err = expandDerivedStars(ctx, prepared, translator, relations); err != nil { return nil, nil, err } + local := fmt.Sprintf("r%d", len(relations)) sourcePlan := *prepared sourcePlan.SelectExprs = make(sqlparser.SelectExprs, len(prepared.SelectExprs)) projected := make(sqlparser.SelectExprs, len(prepared.SelectExprs)) @@ -56,7 +54,10 @@ func prepareMutationRelations( cloned := *expression cloned.As = sqlparser.NewColIdent(field) sourcePlan.SelectExprs[i] = &cloned - projected[i] = &sqlparser.AliasedExpr{Expr: &sqlparser.ColName{Name: sqlparser.NewColIdent(field)}} + projected[i] = &sqlparser.AliasedExpr{Expr: &sqlparser.ColName{ + Name: sqlparser.NewColIdent(field), + Qualifier: sqlparser.TableName{Name: sqlparser.NewTableIdent(local)}, + }} } names, open, err := planDerivedRead(ctx, &sourcePlan, translator, relations) if err != nil { @@ -66,7 +67,6 @@ func prepareMutationRelations( if err != nil { return nil, nil, err } - local := fmt.Sprintf("r%d", len(relations)) handle := UnstablePrefix + "omni_relations.sources." + local relation, err := omnisdk.NewRelation(handle, nil, columns, open) if err != nil { @@ -81,3 +81,17 @@ func prepareMutationRelations( }}}} return &clone, relations, nil } + +func prepareMutationSource(ctx queryContext, source *sqlparser.Select, current string) ( + *sqlparser.Select, map[string]omnisdk.Relation, error, +) { + storedSource, stored, err := prepareStoredRelations(ctx, source) + if err != nil { + return nil, nil, err + } + expanded, err := expandDocumentViews(ctx, storedSource, current) + if err != nil { + return nil, nil, err + } + return resolveDerivedRelations(ctx, expanded, current, 0, stored) +} diff --git a/internal/stackql/intrinsic/omnisdk.go b/internal/stackql/intrinsic/omnisdk.go index a1c9e1c1b..7ff596180 100644 --- a/internal/stackql/intrinsic/omnisdk.go +++ b/internal/stackql/intrinsic/omnisdk.go @@ -14,9 +14,9 @@ import ( "github.com/lib/pq/oid" "github.com/stackql-labs/omnisdk/pkg/omnisdk" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -27,8 +27,6 @@ const defaultBatchSize = 100 const defaultFlushInterval = 50 * time.Millisecond -const omniAll = "all" - func relationName(path string) string { return strings.ReplaceAll(path, ".", "_") } @@ -385,6 +383,13 @@ func selectFunc( node *sqlparser.Select, currentProvider string, ) (func() internaldto.ExecutorOutput, bool) { + prepared, relations, execErr := prepareExecRelations(ctx, node) + if execErr != nil { + return refuse(execErr), true + } + if len(relations) > 0 { + return docSelectFunc(ctx, prepared, currentProvider, relations) + } if execute, handled := storedSelectFunc(ctx, node, currentProvider); handled { return execute, true } @@ -637,64 +642,7 @@ func providerAuthByProvider(ctx queryContext, cloud string) map[string]*omnisdk. return map[string]*omnisdk.Auth{authProviderName(cloud): omnisdkAuth(authCtx)} } -func omnisdkAuth(authCtx *dto.AuthCtx) *omnisdk.Auth { - if authCtx == nil { - return nil - } - auth := &omnisdk.Auth{ - TenancyOCID: authCtx.OciTenancyOCID, - TenancyOCIDEnvVar: authCtx.OciTenancyOCIDEnvVar, - UserOCID: authCtx.OciUserOCID, - UserOCIDEnvVar: authCtx.OciUserOCIDEnvVar, - Fingerprint: authCtx.OciFingerprint, - FingerprintEnvVar: authCtx.OciFingerprintEnvVar, - PrivateKey: authCtx.OciPrivateKey, - PrivateKeyEnvVar: authCtx.OciPrivateKeyEnvVar, - PrivateKeyPath: authCtx.OciPrivateKeyFilePath, - PrivateKeyPathEnvVar: authCtx.OciPrivateKeyFilePathEnvVar, - Passphrase: authCtx.OciPassphrase, - PassphraseEnvVar: authCtx.OciPassphraseEnvVar, - ConfigFilePath: authCtx.OciConfigFilePath, - Profile: authCtx.OciProfile, - - Type: authCtx.Type, - ValuePrefix: authCtx.ValuePrefix, - Name: authCtx.Name, - Scopes: authCtx.Scopes, - TokenURL: authCtx.GetTokenURL(), - Username: authCtx.Username, - Password: authCtx.Password, - UsernameEnvVar: authCtx.EnvVarUsername, - PasswordEnvVar: authCtx.EnvVarPassword, - } - if strings.EqualFold(authCtx.Type, "basic") && auth.ValuePrefix == "" { - // Preserve the CLI's established Basic scheme spelling. - auth.ValuePrefix = "BASIC " - } - if (strings.EqualFold(authCtx.Type, "api_key") || strings.EqualFold(authCtx.Type, "custom")) && auth.Name == "" { - auth.Name = "Authorization" - if auth.ValuePrefix == "" && strings.EqualFold(authCtx.Type, "api_key") { - auth.ValuePrefix = "Bearer " - } - } - if credentials, credErr := authCtx.GetCredentialsBytes(); credErr == nil { - auth.SecretAccessKey = string(credentials) - auth.Credentials = string(credentials) - } - if keyID, keyErr := authCtx.GetKeyIDString(); keyErr == nil { - auth.AccessKeyID = keyID - } - if sessionToken, tokErr := authCtx.GetAwsSessionTokenString(); tokErr == nil { - auth.SessionToken = sessionToken - } - if clientID, idErr := authCtx.GetClientID(); idErr == nil { - auth.ClientID = clientID - } - if clientSecret, secErr := authCtx.GetClientSecret(); secErr == nil { - auth.ClientSecret = clientSecret - } - return auth -} +func omnisdkAuth(authCtx *dto.AuthCtx) *omnisdk.Auth { return dto.OmniAuth(authCtx) } type primedStream struct { inner *rowStream @@ -747,6 +695,8 @@ func (c column) reportedType() string { return columnType case "boolean": return "bool" + case "unknown": + return "" default: return c.dataType } @@ -809,8 +759,6 @@ type backendInput interface { getFlushInterval() time.Duration getInsecureSkipTLSVerify() bool getUnstableEnabled() bool - getStagingEnabled() bool - getOmniAll() bool getIaCStateRoot() string } @@ -820,8 +768,6 @@ type standardBackendInput struct { flushInterval time.Duration insecureSkipTLSVerify bool unstableEnabled bool - stagingEnabled bool - omniAll bool iacStateRoot string } @@ -843,11 +789,6 @@ type previewCfgDTO struct { Endpoint json.RawMessage `json:"endpoint"` InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify"` Unstable bool `json:"unstable"` - // Staging opts SELECTs over document-driven relations into RDBMS staging - // for the SQL omnisdk leaves unapplied, instead of refusing them. - Staging bool `json:"staging"` - // Omni "all" routes every provider through omnisdk, never any-sdk. - Omni string `json:"omni"` // IaCStateRoot is the directory every stackql_preview.iac ledger lives // under. A query's 'state' predicate names a directory inside it, never // one outside, so where state is written stays the operator's choice. @@ -885,8 +826,6 @@ func newBackendInput(cfg previewCfgDTO) backendInput { flushInterval: defaultFlushInterval, insecureSkipTLSVerify: cfg.InsecureSkipTLSVerify, unstableEnabled: cfg.Unstable, - stagingEnabled: cfg.Staging, - omniAll: strings.EqualFold(cfg.Omni, omniAll), iacStateRoot: strings.TrimSpace(cfg.IaCStateRoot), } if cfg.BatchSize > 0 { @@ -908,10 +847,6 @@ func (b *standardBackendInput) getInsecureSkipTLSVerify() bool { return b.insecu func (b *standardBackendInput) getUnstableEnabled() bool { return b.unstableEnabled } -func (b *standardBackendInput) getStagingEnabled() bool { return b.stagingEnabled } - -func (b *standardBackendInput) getOmniAll() bool { return b.omniAll } - func (b *standardBackendInput) getIaCStateRoot() string { return b.iacStateRoot } // sourceKey is the row key a column reads from: its own name, unless an alias diff --git a/internal/stackql/intrinsic/omnisdk_test.go b/internal/stackql/intrinsic/omnisdk_test.go index 08dfbd07c..8b8b78b36 100644 --- a/internal/stackql/intrinsic/omnisdk_test.go +++ b/internal/stackql/intrinsic/omnisdk_test.go @@ -13,10 +13,10 @@ import ( "github.com/lib/pq/oid" "github.com/stackql-labs/omnisdk/pkg/omnisdk" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/output" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type fakeRows struct { @@ -311,10 +311,10 @@ func TestOmnisdkAuthAPIKeyDefaultsMatchCanonicalProviders(t *testing.T) { wantName string wantPrefix string }{ - {in: dto.AuthCtx{Type: "api_key", ValuePrefix: "token "}, wantName: "Authorization", wantPrefix: "token "}, - {in: dto.AuthCtx{Type: "api_key"}, wantName: "Authorization", wantPrefix: "Bearer "}, - {in: dto.AuthCtx{Type: "api_key", Name: "X-Api-Key"}, wantName: "X-Api-Key", wantPrefix: ""}, - {in: dto.AuthCtx{Type: "bearer"}, wantName: "", wantPrefix: ""}, + {in: dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: "api_key", ValuePrefix: "token "}}, wantName: "Authorization", wantPrefix: "token "}, + {in: dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: "api_key"}}, wantName: "Authorization", wantPrefix: "Bearer "}, + {in: dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: "api_key", Name: "X-Api-Key"}}, wantName: "X-Api-Key", wantPrefix: ""}, + {in: dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: "bearer"}}, wantName: "", wantPrefix: ""}, } { authCtx := tc.in got := omnisdkAuth(&authCtx) diff --git a/internal/stackql/intrinsic/operation_completion.go b/internal/stackql/intrinsic/operation_completion.go index c855a7d81..52014a091 100644 --- a/internal/stackql/intrinsic/operation_completion.go +++ b/internal/stackql/intrinsic/operation_completion.go @@ -21,6 +21,10 @@ type awaitedDocQuery struct{ docQuery } func (awaitedDocQuery) awaitsMutation() bool { return true } +func (q awaitedDocQuery) getRelations() map[string]omnisdk.Relation { + return queryRelations(q.docQuery) +} + func mutationAwait(stmt sqlparser.Statement) bool { var comments sqlparser.Comments switch node := stmt.(type) { @@ -47,19 +51,23 @@ type completionRead func(context.Context, string, map[string]any) (omnisdk.Row, type completionWait func(context.Context, time.Duration) error type linkedOperationCompletion struct { - mu sync.Mutex - read completionRead - wait completionWait - writer io.Writer + mu sync.Mutex + read completionRead + wait completionWait + writer io.Writer + readResource bool } func newLinkedOperationCompletion( - read completionRead, wait completionWait, writer io.Writer, + read completionRead, wait completionWait, writer io.Writer, resourceNeeded ...bool, ) omnisdk.MutationCompletion { - return &linkedOperationCompletion{read: read, wait: wait, writer: writer} + return &linkedOperationCompletion{read: read, wait: wait, writer: writer, + readResource: len(resourceNeeded) == 0 || resourceNeeded[0]} } -func newOperationCompletion(ctx queryContext, address string) (omnisdk.MutationCompletion, error) { +func newOperationCompletion( + ctx queryContext, address string, resourceNeeded ...bool, +) (omnisdk.MutationCompletion, error) { parts := strings.Split(address, ".") if len(parts) != 3 || strings.TrimPrefix(parts[0], UnstablePrefix) != "google" || parts[1] != "compute" { return nil, fmt.Errorf("AWAIT completion is not supported for %s", address) @@ -90,7 +98,7 @@ func newOperationCompletion(ctx queryContext, address string) (omnisdk.MutationC return nil } } - return newLinkedOperationCompletion(read, wait, writer), nil + return newLinkedOperationCompletion(read, wait, writer, resourceNeeded...), nil } func (c *linkedOperationCompletion) Complete( @@ -139,15 +147,28 @@ func (c *linkedOperationCompletion) Complete( return nil, fmt.Errorf("AWAIT operation failed: %v", failure) } c.diagnosticf("%s: %s complete\n", kind, action) - if verb == "delete" { + return c.completedResult(ctx, verb, action, target, inputs, operation) +} + +func (c *linkedOperationCompletion) completedResult( + ctx context.Context, verb, action, target string, inputs map[string]any, operation omnisdk.Row, +) (omnisdk.Row, error) { + if operationDeletesResource(verb, action) { return nil, nil //nolint:nilnil // Successful DELETE has no resource row. } + if !c.readResource { + return operation, nil + } if target == "" { return nil, fmt.Errorf("AWAIT completed without targetLink") } return c.read(ctx, target, inputs) } +func operationDeletesResource(verb, action string) bool { + return verb == "delete" || (verb == "exec" && action == "delete") +} + func (c *linkedOperationCompletion) diagnosticf(format string, values ...any) { c.mu.Lock() defer c.mu.Unlock() @@ -222,7 +243,8 @@ func configureMutationCompletion(ctx queryContext, translated docQuery, args *om if !queryAwaitsMutation(translated) { return nil } - completion, err := newOperationCompletion(ctx, translated.getQuery().Target().Resource().Handle()) + completion, err := newOperationCompletion( + ctx, translated.getQuery().Target().Resource().Handle(), len(translated.getOutputs()) > 0) if err != nil { return err } diff --git a/internal/stackql/intrinsic/preview_services.go b/internal/stackql/intrinsic/preview_services.go index c8904bc96..26e8520fd 100644 --- a/internal/stackql/intrinsic/preview_services.go +++ b/internal/stackql/intrinsic/preview_services.go @@ -20,10 +20,9 @@ import ( "sync" "github.com/stackql-labs/omnisdk/pkg/omnisdk" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql-parser/go/vt/sqlparser" @@ -60,7 +59,7 @@ const ( // reports false. They read documents from disk, so they appear only once that // was opted into, exactly as the unstable providers do. func previewService(providerName, serviceStr, currentProvider string) (string, bool) { - if !IsUnstableEnabled() { + if !previewCfg.getUnstableEnabled() { return "", false } resolved := strings.TrimSpace(resolveProvider(providerName, currentProvider)) @@ -81,7 +80,7 @@ func previewService(providerName, serviceStr, currentProvider string) (string, b // It is a no-op without the opt-in, which keeps the catalogue exactly as it was // for anyone who has not asked for them. func appendPreviewServices(rows map[string]map[string]interface{}, extended bool) { - if !IsExtendedCatalogEnabled() { + if !previewCfg.getUnstableEnabled() { return } for i, svc := range []struct{ name, title string }{ @@ -108,7 +107,7 @@ func appendPreviewServices(rows map[string]map[string]interface{}, extended bool func registryRoot(ctx queryContext) string { runtime := ctx.GetRuntimeContext() prefix := "src" - var cfg formulation.RegistryConfig + var cfg omnisdk.RegistryConfig if err := yaml.Unmarshal([]byte(runtime.RegistryRaw), &cfg); err == nil && cfg.SrcPrefix != nil { prefix = *cfg.SrcPrefix } @@ -171,7 +170,7 @@ func previewArgs(ctx queryContext, cloud string, params map[string]string) (omni return omnisdk.Args{}, err } policy := newReadErrorPolicy(ctx, upstream.HTTP) - return omnisdk.Args{ + args := omnisdk.Args{ DocumentVerifier: verifier, Params: params, AuthByProvider: providerAuthByProvider(ctx, cloud), @@ -189,7 +188,8 @@ func previewArgs(ctx queryContext, cloud string, params map[string]string) (omni RequestTimeout: apiRequestTimeout(ctx), PageLimit: ctx.GetRuntimeContext().HTTPPageLimit, }, - }, nil + } + return args, nil } // previewSelectFunc routes a SELECT over an extended preview relation. @@ -274,7 +274,7 @@ func showPreviewResources( } rows[fmt.Sprintf("%06d", i)] = row } - return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort) + return prepare(ctx, util.GetResourcesHeader(extended), rows, util.DefaultRowSort) } func showPreviewMethods(ctx queryContext, extended bool) internaldto.ExecutorOutput { @@ -358,6 +358,7 @@ func describePreviewTableFunc( type readErrorPolicy struct { dto.HTTPUpstreamErrorCfg writer io.Writer + strict bool mu sync.Mutex } @@ -366,12 +367,17 @@ func newReadErrorPolicy(ctx queryContext, config dto.HTTPUpstreamErrorCfg) *read if output, ok := ctx.(interface{ GetOutErrFile() io.Writer }); ok { writer = output.GetOutErrFile() } - return &readErrorPolicy{HTTPUpstreamErrorCfg: config, writer: writer} + return &readErrorPolicy{ + HTTPUpstreamErrorCfg: config, writer: writer, strict: ctx.GetRuntimeContext().StrictUpstreamErrors, + } } // StackQL presents absent provider resources as empty tables. This host policy // applies only to resource reads; SDK mutation and auth exchanges remain strict. func (p *readErrorPolicy) IsIgnored(status int) bool { + if p.strict { + return status == http.StatusNotFound || p.HTTPUpstreamErrorCfg.IsIgnored(status) + } return status >= http.StatusBadRequest && status <= 599 || p.HTTPUpstreamErrorCfg.IsIgnored(status) } func (p *readErrorPolicy) IgnoredStatus(status int, body string) { diff --git a/internal/stackql/intrinsic/read_absence_test.go b/internal/stackql/intrinsic/read_absence_test.go index b6a644510..ae27555b1 100644 --- a/internal/stackql/intrinsic/read_absence_test.go +++ b/internal/stackql/intrinsic/read_absence_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) func TestReadFailuresRemainLocalAndPreserveDiagnostics(t *testing.T) { diff --git a/internal/stackql/intrinsic/registry_metadata.go b/internal/stackql/intrinsic/registry_metadata.go index 15faac767..4bed2867b 100644 --- a/internal/stackql/intrinsic/registry_metadata.go +++ b/internal/stackql/intrinsic/registry_metadata.go @@ -1,30 +1,15 @@ package intrinsic -import "github.com/stackql/stackql/internal/stackql/provider" +import "github.com/stackql-labs/omnisdk/pkg/omnisdk" -// Execution-backend selection does not alter the registry's presentation -// metadata. Keep resource keys for routing, and declared names for SHOW output. -func registryResourceNames(ctx queryContext, bundle, service string, extended bool) (map[string]string, error) { - if !previewCfg.getOmniAll() { - return map[string]string{}, nil - } - host, ok := ctx.(interface { - GetProvider(string) (provider.IProvider, error) - }) - if !ok { - return map[string]string{}, nil - } - prov, err := host.GetProvider(bundle) +func registryResourceNames(ctx queryContext, bundle, service string, _ bool) (map[string]string, error) { + dir, err := docRoot(ctx, bundle) if err != nil { return nil, err } - resources, err := prov.GetResourcesRedacted(service, ctx.GetRuntimeContext(), extended) + opts, err := documentOptions(ctx) if err != nil { return nil, err } - names := make(map[string]string, len(resources)) - for key, resource := range resources { - names[key] = resource.GetName() - } - return names, nil + return omnisdk.DocResourceNames(dir, bundle, service, opts...) } diff --git a/internal/stackql/intrinsic/relation_reads.go b/internal/stackql/intrinsic/relation_reads.go index 0112dce31..c579d999b 100644 --- a/internal/stackql/intrinsic/relation_reads.go +++ b/internal/stackql/intrinsic/relation_reads.go @@ -3,9 +3,11 @@ package intrinsic import ( "context" "database/sql" + "encoding/json" "fmt" "maps" "sort" + "strings" "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql/stackql-parser/go/vt/sqlparser" @@ -97,30 +99,10 @@ func resolveDerivedTable( if !isSubquery { return nil } - inner, isSelect := sub.Select.(*sqlparser.Select) - if !isSelect { - return fmt.Errorf("derived relation requires SELECT") - } if table.As.IsEmpty() { return fmt.Errorf("derived relation requires an alias") } - expanded, err := expandDocumentViews(ctx, inner, current) - if err != nil { - return err - } - prepared, children, err := resolveDerivedRelations(ctx, expanded, current, depth+1, maps.Clone(relations)) - if err != nil { - return err - } - dialect, err := backendDialect(ctx) - if err != nil { - return err - } - translator := newDocTranslator(current, dialect) - if err = expandDerivedStars(ctx, prepared, translator, children); err != nil { - return err - } - names, open, err := planDerivedRead(ctx, prepared, translator, children) + names, open, err := planStatementRead(ctx, sub.Select, current, depth+1, maps.Clone(relations)) if err != nil { return err } @@ -134,7 +116,7 @@ func resolveDerivedTable( if err != nil { return err } - relations[handle] = relation + relations[handle] = newOrderedRelation(relation, names) table.Expr = sqlparser.TableName{ QualifierSecond: sqlparser.NewTableIdent(UnstablePrefix + "omni_relations"), Qualifier: sqlparser.NewTableIdent("sources"), @@ -152,14 +134,14 @@ func planDerivedRead( return planBackendRead(ctx, node) } if needsStaging(node) { - if !previewCfg.getStagingEnabled() { - return nil, nil, fmt.Errorf("derived relation requires staging") - } staged, err := planStagedSelect(node, translator, ctx.GetASTFormatter()) if err != nil { return nil, nil, err } staged = withStagedRelations(staged, children) + if _, _, planErr := planDocQuery(ctx, staged.getSource()); planErr != nil { + return nil, nil, planErr + } open := func(runCtx context.Context, _ map[string]any) (omnisdk.Rows, error) { if cancelErr := runCtx.Err(); cancelErr != nil { return nil, cancelErr @@ -173,9 +155,12 @@ func planDerivedRead( return nil, nil, err } translated = withRelations(translated, children) + plan, _, err := planDocQuery(ctx, translated) + if err != nil { + return nil, nil, err + } open := func(runCtx context.Context, _ map[string]any) (omnisdk.Rows, error) { - rows, _, openErr := openDocQueryIn(ctx, translated, runCtx) - return rows, openErr + return plan.Open(runCtx) } return translated.getOutputs(), open, nil } @@ -229,6 +214,10 @@ func materializeSQLRows( defer result.Close() var err error var rows []omnisdk.Row + numeric, err := numericSQLColumns(result) + if err != nil { + return nil, err + } for result.Next() { values := make([]any, len(names)) pointers := make([]any, len(names)) @@ -240,11 +229,7 @@ func materializeSQLRows( } row := omnisdk.Row{} for i, name := range names { - value := values[i] - if raw, ok := value.([]byte); ok { - value = string(raw) - } - row[name] = value + row[name] = nativeSQLValue(values[i], numeric[i]) } rows = append(rows, row) } @@ -371,6 +356,9 @@ func declaredRelationColumns( ) ([]string, error) { var table omnisdk.Table if relation, present := relations[handle]; present { + if ordered, ok := relation.(orderedRelation); ok { + return ordered.columnOrder(), nil + } table = relation } else { opts, err := documentOptions(ctx) @@ -440,3 +428,31 @@ func planBackendRead(ctx queryContext, node *sqlparser.Select) ([]string, derive } return names, open, nil } + +func numericSQLColumns(rows *sql.Rows) ([]bool, error) { + types, err := rows.ColumnTypes() + if err != nil { + return nil, err + } + numeric := make([]bool, len(types)) + for i, column := range types { + switch strings.ToLower(column.DatabaseTypeName()) { + case "numeric", "decimal", "integer", "int", "int2", "int4", "int8", "bigint", "smallint", + "float4", "float8", "real", "double precision": + numeric[i] = true + } + } + return numeric, nil +} + +// Database drivers may return NUMERIC as bytes or text; retain its numeric +// identity when rows cross back into Omni for a parent query. +func nativeSQLValue(value any, numeric bool) any { + if raw, ok := value.([]byte); ok { + value = string(raw) + } + if text, ok := value.(string); ok && numeric { + return json.Number(text) + } + return value +} diff --git a/internal/stackql/intrinsic/server_environment.go b/internal/stackql/intrinsic/server_environment.go index 89a09723f..e9e9d1dd4 100644 --- a/internal/stackql/intrinsic/server_environment.go +++ b/internal/stackql/intrinsic/server_environment.go @@ -84,3 +84,27 @@ func unresolvedServerParams(method omnisdk.DocMethod) []string { } return required } + +// bindReadPageSize retains the CLI's request size independently of a SQL LIMIT. +// Only declared query parameters are supplied, and explicit SQL bindings win. +func bindReadPageSize(q query.Unresolved, tables map[string]omnisdk.Table, size int) (query.Unresolved, error) { + if size <= 0 { + return q, nil + } + where := slices.Clone(q.Where()) + for _, alias := range slices.Sorted(maps.Keys(tables)) { + if q.Target() != nil && q.Target().Resource().Alias() == alias { + continue + } + for _, method := range tables[alias].Methods() { + for _, parameter := range method.Params() { + if parameter.In() == "query" && parameter.Name() == "maxResults" && !explicitServerBinding(q, alias, "maxResults") { + where = append(where, query.NewEq(query.NewColumn(alias, "maxResults"), query.NewLiteral(size))) + break + } + } + break + } + } + return queryWithWhere(q, where) +} diff --git a/internal/stackql/intrinsic/show_insert.go b/internal/stackql/intrinsic/show_insert.go new file mode 100644 index 000000000..998530905 --- /dev/null +++ b/internal/stackql/intrinsic/show_insert.go @@ -0,0 +1,41 @@ +package intrinsic + +import ( + "fmt" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" + "github.com/stackql/stackql/internal/stackql/metadatavisitors" + "github.com/stackql/stackql/internal/stackql/util" + "github.com/stackql/stackql/pkg/prettyprint" +) + +func showDocInsert(ctx queryContext, node *sqlparser.Show, bundle string) internaldto.ExecutorOutput { + opts, err := documentOptions(ctx) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + address := UnstablePrefix + bundle + "." + node.OnTable.Qualifier.GetRawVal() + "." + node.OnTable.Name.GetRawVal() + table, err := omnisdk.DescribeMutation(registryRoot(ctx), address, "insert", opts...) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + if len(table.Methods()) == 0 { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("%s declares no INSERT method", address)) + } + context := prettyprint.NewPrettyPrintContext( + ctx.GetRuntimeContext().OutputFormat == constants.PrettyTextStr, + constants.DefaultPrettyPrintIndent, constants.DefaultPrettyPrintBaseIndent, "'", logging.GetLogger()) + statement, err := metadatavisitors.ToOmniInsertStatement(node.Columns, table.Methods()[0], isExtended(node.Extended), + sqlparser.ExtractCommentDirectives(node.Comments).IsSet("REQUIRED"), + prettyprint.NewPrettyPrinter(context), prettyprint.NewPrettyPrinter(context)) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + return prepare(ctx, []string{"insert_statement"}, map[string]map[string]interface{}{ + "1": {"insert_statement": fmt.Sprintf(statement, qualifiedName(node.OnTable))}, + }, util.DefaultRowSort) +} diff --git a/internal/stackql/intrinsic/sleep.go b/internal/stackql/intrinsic/sleep.go new file mode 100644 index 000000000..ce62b77d5 --- /dev/null +++ b/internal/stackql/intrinsic/sleep.go @@ -0,0 +1,34 @@ +package intrinsic + +import ( + "context" + "fmt" + "time" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/parserutil" +) + +func sleepFunc(node *sqlparser.Sleep) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + duration, err := parserutil.ExtractSleepDuration(node) + if err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + planned, err := omnisdk.NewSleep(time.Duration(duration) * time.Millisecond) + if err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + rows, err := planned.Open(context.Background()) + if err != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) + } + defer rows.Close() + for rows.Next() { + } + return internaldto.NewExecutorOutput(nil, nil, nil, + internaldto.NewBackendMessages([]string{fmt.Sprintf("Success: slept for %d milliseconds", duration)}), rows.Err()) + } +} diff --git a/internal/stackql/intrinsic/staged.go b/internal/stackql/intrinsic/staged.go index 45184ac9b..d98ad28ba 100644 --- a/internal/stackql/intrinsic/staged.go +++ b/internal/stackql/intrinsic/staged.go @@ -2,7 +2,7 @@ package intrinsic // A SELECT over document-driven relations whose remaining SQL needs every row // - ordering, grouping, aggregation, de-duplication, OFFSET - is split in two -// when staging is enabled: omnisdk runs the joins and filters, its final +// omnisdk runs the joins and filters, its final // relation is staged in one query-owned table, and the RDBMS evaluates the // rest of the statement over that table. See docs/technical/omnisdk_staging.md. @@ -15,14 +15,14 @@ import ( "io" "strconv" "strings" - "sync" + "github.com/lib/pq/oid" "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/omnistaging" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql-parser/go/vt/sqlparser" @@ -41,42 +41,26 @@ const ( stagedNumericType = "numeric" ) -// stagingManager is created on first use from the SQL backend the session -// already holds, and shared by every query thereafter. -// -//nolint:gochecknoglobals // one manager per backend, created once -var ( - stagingOnce sync.Once - stagingManager omnistaging.Manager - errStaging error -) - +// Each query owns its staging manager and uses the session's SQL backend. func getStagingManager(ctx queryContext) (omnistaging.Manager, error) { - stagingOnce.Do(func() { - raw := ctx.GetRuntimeContext().SQLBackendCfgRaw - sqlCfg, err := dto.GetSQLBackendCfg(raw) - if err != nil { - errStaging = err - return - } - cfg, err := omnistaging.NewConfigFromSQLBackend(raw) - if err != nil { - errStaging = err - return - } - dialect, err := omnistaging.NewDialect(sqlCfg.GetSQLDialect(), cfg) - if err != nil { - errStaging = err - return - } - db, err := ctx.GetSQLEngine().GetDB() - if err != nil { - errStaging = err - return - } - stagingManager, errStaging = omnistaging.NewManager(context.Background(), db, dialect) - }) - return stagingManager, errStaging + raw := ctx.GetRuntimeContext().SQLBackendCfgRaw + sqlCfg, err := dto.GetSQLBackendCfg(raw) + if err != nil { + return nil, err + } + cfg, err := omnistaging.NewConfigFromSQLBackend(raw) + if err != nil { + return nil, err + } + dialect, err := omnistaging.NewDialect(sqlCfg.GetSQLDialect(), cfg) + if err != nil { + return nil, err + } + db, err := ctx.GetSQLEngine().GetDB() + if err != nil { + return nil, err + } + return omnistaging.NewManager(context.Background(), db, dialect) } // needsStaging reports whether a SELECT carries SQL omnisdk leaves unapplied. @@ -90,6 +74,10 @@ func needsStaging(node *sqlparser.Select) bool { hasAggregate := false //nolint:errcheck // the visitor returns no error _ = sqlparser.Walk(func(n sqlparser.SQLNode) (bool, error) { + if _, groupConcat := n.(*sqlparser.GroupConcatExpr); groupConcat { + hasAggregate = true + return false, nil + } if fn, isFunc := n.(*sqlparser.FuncExpr); isFunc && (fn.IsAggregate() || fn.Over != nil) { hasAggregate = true return false, nil @@ -584,6 +572,17 @@ func readStagedResult( ) (internaldto.ExecutorOutput, error) { var err error rowMap := make(map[string]map[string]interface{}) + numeric, err := numericSQLColumns(result) + if err != nil { + return nil, err + } + columnOIDs := make([]oid.Oid, len(columnOrder)) + for i := range columnOIDs { + columnOIDs[i] = oid.T_text + if numeric[i] { + columnOIDs[i] = oid.T_numeric + } + } for i := 0; result.Next(); i++ { values := make([]any, len(columnOrder)) pointers := make([]any, len(columnOrder)) @@ -596,13 +595,19 @@ func readStagedResult( row := make(map[string]interface{}, len(columnOrder)) for j, name := range columnOrder { row[name] = outputValue(values[j]) + if isNumber(values[j]) { + columnOIDs[j] = oid.T_numeric + } } rowMap[fmt.Sprintf("%012d", i)] = row } if err = result.Err(); err != nil { return nil, err } - return prepare(ctx, columnOrder, rowMap, util.DefaultRowSort), nil + payload := internaldto.NewPrepareResultSetDTO( + nil, rowMap, columnOrder, util.DefaultRowSort, nil, nil, ctx.GetTypingConfig()) + payload.ColumnOIDs = columnOIDs + return util.PrepareResultSet(payload), nil } func outputValue(value any) any { diff --git a/internal/stackql/intrinsic/staged_test.go b/internal/stackql/intrinsic/staged_test.go index 2a95a4a80..fa8101410 100644 --- a/internal/stackql/intrinsic/staged_test.go +++ b/internal/stackql/intrinsic/staged_test.go @@ -8,14 +8,15 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "testing" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/output" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/pkg/astformat" @@ -141,7 +142,7 @@ func newStagingTestCtx(t *testing.T, endpoint string) *stagingTestCtx { if err != nil { t.Fatal(err) } - engine, err := sqlengine.NewSQLEngine(sqlCfg, nil) + engine, err := sqlengine.NewSQLEngine(sqlengine.Config{DBEngine: sqlCfg.DBEngine, DSN: sqlCfg.GetDSN()}, nil) if err != nil { t.Fatal(err) } @@ -160,7 +161,7 @@ func newStagingTestCtx(t *testing.T, endpoint string) *stagingTestCtx { } previous := previewCfg previewCfg = newBackendInput(previewCfgDTO{ - Unstable: true, Staging: true, Endpoint: endpointJSON, InsecureSkipTLSVerify: true, + Unstable: true, Endpoint: endpointJSON, InsecureSkipTLSVerify: true, }) t.Cleanup(func() { previewCfg = previous }) return &stagingTestCtx{ @@ -193,8 +194,28 @@ func renderCSV(t *testing.T, sql string, out internaldto.ExecutorOutput) string return buf.String() } -// The staging manager is created once per process, so every end-to-end case -// shares one backend and runs in this one test. +func TestDescribeDocumentWithoutExecutingProvider(t *testing.T) { + ctx := newStagingTestCtx(t, "https://unused.invalid") + for _, sql := range []string{ + "describe fixture.orgs.members", + "describe method fixture.orgs.members.list_members", + } { + statement, err := sqlparser.Parse(sql) + if err != nil { + t.Fatal(err) + } + execute, handled := GeneratePrimitiveFunc(ctx, statement) + if !handled { + t.Fatalf("metadata statement was not handled: %s", sql) + } + result := renderCSV(t, sql, execute()) + if !strings.Contains(result, "login,string") || !strings.Contains(result, "id,integer") { + t.Fatalf("missing declared response columns: %s", result) + } + } +} + +// These end-to-end cases share a backend to verify staging cleanup across queries. func TestStagedSelectEndToEnd(t *testing.T) { members := []map[string]any{ {"login": "a", "id": 1, "type": "User"}, @@ -404,6 +425,9 @@ func TestStagedSelectEndToEnd(t *testing.T) { } } + testMaterializedReads(t, ctx) + testStatementReads(t, ctx) + db, err := ctx.engine.GetDB() if err != nil { t.Fatal(err) @@ -428,3 +452,67 @@ func TestExpandedCTEDepthHonorsHostLimit(t *testing.T) { t.Fatal(err) } } + +func testMaterializedReads(t *testing.T, ctx *stagingTestCtx) { + t.Helper() + for _, query := range []string{ + "create materialized view omni_members as select login, id from fixture.orgs.members where org = 'dummyorg' order by id", + "create or replace materialized view omni_members as select login, id from fixture.orgs.members where org = 'dummyorg' order by id", + } { + execute, claimed := materializedFunc(ctx, parseStatement(t, query)) + if !claimed { + t.Fatalf("materialized query not claimed: %s", query) + } + if err := execute().GetError(); err != nil { + t.Fatal(err) + } + } + read, claimed := selectFunc(ctx, parseSelect(t, "select count(*) as n from omni_members"), "") + if !claimed { + t.Fatal("stored materialized view not claimed") + } + if got := renderCSV(t, "select count(*) as n from omni_members", read()); got != "n\n4\n" { + t.Fatalf("materialized rows: %q", got) + } + +} + +func testStatementReads(t *testing.T, ctx *stagingTestCtx) { + t.Helper() + for _, tc := range []struct{ sql, want string }{ + {"select 1 as n union all select 1 as n", "n\n1\n1\n"}, + {"select 1 as n union select 1 as n", "n\n1\n"}, + {"select login, id from fixture.orgs.members where org = 'dummyorg' union all select login, id from fixture.orgs.members where org = 'dummyorg' order by id desc limit 2", "login,id\nb,10\nb,10\n"}, + {"with combined as (select login from fixture.orgs.members where org = 'dummyorg' union all select login from fixture.orgs.members where org = 'dummyorg') select count(*) as n from combined", "n\n8\n"}, + } { + execute, handled := GenerateStreamFunc(ctx, parseStatement(t, tc.sql)) + if !handled { + t.Fatalf("statement not claimed: %s", tc.sql) + } + if got := renderCSV(t, tc.sql, execute()); got != tc.want { + t.Errorf("%s: got %q, want %q", tc.sql, got, tc.want) + } + } +} + +func TestValidateStatementDoesNotDispatchRequests(t *testing.T) { + t.Setenv("FIXTURE_TOKEN", "test-token") + ctx := newStagingTestCtx(t, "https://unused.invalid") + for _, test := range []struct { + sql string + valid bool + }{ + {"select login from fixture.orgs.members where org = 'dummyorg'", true}, + {"select count(*) from fixture.orgs.members where org = 'dummyorg'", true}, + {"select login from fixture.orgs.missing", false}, + } { + statement, err := sqlparser.Parse(test.sql) + if err != nil { + t.Fatal(err) + } + handled, err := ValidateStatement(ctx, statement) + if !handled || (err == nil) != test.valid { + t.Fatalf("%s: handled=%v err=%v", test.sql, handled, err) + } + } +} diff --git a/internal/stackql/intrinsic/star_schema.go b/internal/stackql/intrinsic/star_schema.go index 4480a1055..d2ad0fc20 100644 --- a/internal/stackql/intrinsic/star_schema.go +++ b/internal/stackql/intrinsic/star_schema.go @@ -39,6 +39,14 @@ func resolvedStarColumns(ctx queryContext, source translatedSource, star *sqlpar if selected, ok := resolved.(omnisdk.SelectedMethods); ok { methods = selected.SelectedMethods() } + if methods == nil { + methods = map[string]omnisdk.MethodSignature{} + } + for alias, table := range tables { + if _, ordered := table.(orderedRelation); ordered { + methods[alias] = table.Methods()[0] + } + } excluded := map[string]bool{} for _, output := range aliasOutputs { excluded[output.Name()] = true @@ -91,7 +99,11 @@ func starSQLColumns( } // Scope columns describe host output; they do not introduce value producers. names = append(names, starScopeNames(methods[projection.Alias()])...) - slices.Sort(names) + if method, ok := methods[projection.Alias()].(orderedSignature); ok { + names = method.Columns() + } else { + slices.Sort(names) + } for _, name := range slices.Compact(names) { expressions = append(expressions, &sqlparser.AliasedExpr{Expr: &sqlparser.ColName{ Name: sqlparser.NewColIdent(name), diff --git a/internal/stackql/intrinsic/statement_reads.go b/internal/stackql/intrinsic/statement_reads.go new file mode 100644 index 000000000..ef3bba2a6 --- /dev/null +++ b/internal/stackql/intrinsic/statement_reads.go @@ -0,0 +1,158 @@ +package intrinsic + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/psql-wire/pkg/sqldata" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" +) + +// The host coordinates statement boundaries; provider reads are always SDK plans. +func planStatementRead( + ctx queryContext, stmt sqlparser.SelectStatement, current string, depth int, + inherited map[string]omnisdk.Relation, +) ([]string, derivedOpener, error) { + if depth >= derivedRelationMaxDepth { + return nil, nil, fmt.Errorf("derived relations exceed nesting bound") + } + switch node := stmt.(type) { + case *sqlparser.ParenSelect: + return planStatementRead(ctx, node.Select, current, depth+1, inherited) + case *sqlparser.Union: + return planUnionRead(ctx, node, current, depth+1, inherited) + case *sqlparser.Select: + prepared, stored, err := prepareStoredRelations(ctx, node) + if err != nil { + return nil, nil, err + } + if stored == nil { + stored = map[string]omnisdk.Relation{} + } + for handle, relation := range inherited { + stored[handle] = relation + } + expanded, err := expandDocumentViews(ctx, prepared, current) + if err != nil { + return nil, nil, err + } + prepared, children, err := resolveDerivedRelations(ctx, expanded, current, depth+1, stored) + if err != nil { + return nil, nil, err + } + dialect, err := backendDialect(ctx) + if err != nil { + return nil, nil, err + } + translator := newDocTranslator(current, dialect) + if err = expandDerivedStars(ctx, prepared, translator, children); err != nil { + return nil, nil, err + } + return planDerivedRead(ctx, prepared, translator, children) + } + return nil, nil, fmt.Errorf("unsupported read statement %T", stmt) +} + +func planUnionRead( + ctx queryContext, node *sqlparser.Union, current string, depth int, inherited map[string]omnisdk.Relation, +) ([]string, derivedOpener, error) { + statements := []sqlparser.SelectStatement{node.FirstStatement} + for _, branch := range node.UnionSelects { + statements = append(statements, branch.Statement) + } + names := make([][]string, len(statements)) + opens := make([]derivedOpener, len(statements)) + for i, statement := range statements { + var err error + names[i], opens[i], err = planStatementRead(ctx, statement, current, depth+1, inherited) + if err != nil { + return nil, nil, err + } + if i > 0 && len(names[i]) != len(names[0]) { + return nil, nil, fmt.Errorf("UNION branches have different column counts") + } + } + open := func(runCtx context.Context, _ map[string]any) (omnisdk.Rows, error) { + return openUnionRead(ctx, runCtx, node, names, opens) + } + return names[0], open, nil +} + +func openUnionRead( + ctx queryContext, runCtx context.Context, node *sqlparser.Union, names [][]string, opens []derivedOpener, +) (omnisdk.Rows, error) { + manager, err := getStagingManager(ctx) + if err != nil { + return nil, err + } + var statements []string + for i, open := range opens { + id, beginErr := manager.Begin(runCtx) + if beginErr != nil { + return nil, beginErr + } + defer manager.Release(context.Background(), id) //nolint:errcheck // Preserve the query outcome. + rows, openErr := open(runCtx, nil) + if openErr != nil { + return nil, openErr + } + source := newOmniBatchSource(rows, names[i], previewCfg.getBatchSize()) + first, readErr := source.Next(runCtx) + if readErr != nil && !errors.Is(readErr, io.EOF) { + rows.Close() + return nil, readErr + } + source.pending = first + _, stageErr := manager.Stage(runCtx, id, stagedColumns(names[i], first), source) + rows.Close() + if stageErr != nil { + return nil, stageErr + } + if source.terminalErr != nil { + return nil, source.terminalErr + } + if i > 0 { + statements = append(statements, strings.ToUpper(node.UnionSelects[i-1].Type)) + } + statements = append(statements, "SELECT * FROM "+manager.TableName(id)) + } + formatter := sqlparser.NewTrackedBuffer(ctx.GetASTFormatter()) + formatter.Myprintf("%v%v", node.OrderBy, node.Limit) + // Table names are generated locally and the suffix is rendered from the parsed SQL AST. + statement := strings.Join(statements, " ") + formatter.String() // #nosec G202 + db, err := ctx.GetSQLEngine().GetDB() + if err != nil { + return nil, err + } + result, err := db.QueryContext(runCtx, statement) + if err != nil { + return nil, err + } + defer result.Close() + return materializeSQLRows(runCtx, result, names[0], nil) +} + +func statementReadFunc(ctx queryContext, stmt sqlparser.SelectStatement) func() internaldto.ExecutorOutput { + names, open, err := planStatementRead(ctx, stmt, ctx.GetCurrentProvider(), 0, nil) + if err != nil { + return refuse(err) + } + return func() internaldto.ExecutorOutput { + rows, openErr := open(context.Background(), nil) + if openErr != nil { + return internaldto.NewErroneousExecutorOutput(openErr) + } + stream := &rowStream{rows: rows, batchSize: previewCfg.getBatchSize(), flushInterval: previewCfg.getFlushInterval(), + table: sqldata.NewSQLTable(0, "union"), typCfg: ctx.GetTypingConfig(), outputs: names} + primed, readErr := newPrimedStream(stream) + if readErr != nil { + return internaldto.NewErroneousExecutorOutput(readErr) + } + return internaldto.NewExecutorOutput(primed, nil, nil, nil, nil) + } +} diff --git a/internal/stackql/intrinsic/stored_insert.go b/internal/stackql/intrinsic/stored_insert.go index d6c7b412f..9b2d738bb 100644 --- a/internal/stackql/intrinsic/stored_insert.go +++ b/internal/stackql/intrinsic/stored_insert.go @@ -16,11 +16,11 @@ import ( // follows the same Omni planning and staging path as an ordinary SELECT. func storedInsertSelectFunc(ctx queryContext, stmt sqlparser.Statement) (func() internaldto.ExecutorOutput, bool) { insert, ok := stmt.(*sqlparser.Insert) - if !ok || !previewCfg.getOmniAll() { + if !ok { return nil, false } source, ok := insert.Rows.(*sqlparser.Select) - if !ok || !containsProviderRead(source, ctx.GetCurrentProvider(), nil) { + if !ok || literalSource(source.From) { return nil, false } host, ok := ctx.(interface{ GetSQLSystem() sql_system.SQLSystem }) @@ -36,10 +36,7 @@ func storedInsertSelectFunc(ctx queryContext, stmt sqlparser.Statement) (func() if insert.Ignore != "" || len(insert.OnDup) > 0 { return refuse(fmt.Errorf("host table INSERT source does not support conflict clauses")), true } - target, err := system.GetFullyQualifiedTableName(table.GetName()) - if err != nil { - return refuse(err), true - } + target := system.DelimitFullyQualifiedRelationName(table.GetName()) prepared, relations, err := prepareMutationRelations(ctx, insert, ctx.GetCurrentProvider()) if err != nil { return refuse(err), true diff --git a/internal/stackql/intrinsic/stored_relations.go b/internal/stackql/intrinsic/stored_relations.go index 045a9260b..201898163 100644 --- a/internal/stackql/intrinsic/stored_relations.go +++ b/internal/stackql/intrinsic/stored_relations.go @@ -22,7 +22,49 @@ type storedResolver struct { system sql_system.SQLSystem relations map[string]omnisdk.Relation active map[string]bool - ctes map[string]*sqlparser.Select + ctes map[string]sqlparser.SelectStatement +} + +func describeStoredFunc( + ctx queryContext, node *sqlparser.DescribeTable, current string, +) (func() internaldto.ExecutorOutput, bool) { + host, ok := ctx.(interface{ GetSQLSystem() sql_system.SQLSystem }) + if !ok || host.GetSQLSystem() == nil { + return nil, false + } + system := host.GetSQLSystem() + name := qualifiedName(node.Table) + _, view := system.GetViewByName(name) + materialized, mv := system.GetMaterializedViewByName(name) + physical, tableExists := system.GetPhysicalTableByName(name) + if !view && !mv && !tableExists { + return nil, false + } + return func() internaldto.ExecutorOutput { + var columns []column + if view { + selectNode := &sqlparser.Select{ + SelectExprs: sqlparser.SelectExprs{&sqlparser.StarExpr{}}, + From: sqlparser.TableExprs{&sqlparser.AliasedTableExpr{Expr: node.Table}}, + } + names, _, err := planStatementRead(ctx, selectNode, current, 0, nil) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + for _, name := range names { + columns = append(columns, column{name: name}) + } + } else { + metadata := physical + if mv { + metadata = materialized + } + for _, declared := range metadata.GetColumns() { + columns = append(columns, column{name: declared.GetName(), dataType: declared.GetType()}) + } + } + return describeTable(ctx, table{columns: columns}, isExtended(node.Extended)) + }, true } func newStoredResolver(ctx queryContext, system sql_system.SQLSystem) storedRelationResolver { @@ -31,9 +73,6 @@ func newStoredResolver(ctx queryContext, system sql_system.SQLSystem) storedRela func prepareStoredRelations( ctx queryContext, node *sqlparser.Select, ) (*sqlparser.Select, map[string]omnisdk.Relation, error) { - if !IsUnstableEnabled() { - return node, nil, nil - } var system sql_system.SQLSystem if host, ok := ctx.(interface{ GetSQLSystem() sql_system.SQLSystem }); ok { system = host.GetSQLSystem() @@ -95,12 +134,16 @@ func (r *storedResolver) selectNode(node *sqlparser.Select, depth int) (*sqlpars parent := r.ctes r.ctes = maps.Clone(parent) if r.ctes == nil { - r.ctes = map[string]*sqlparser.Select{} + r.ctes = map[string]sqlparser.SelectStatement{} } defer func() { r.ctes = parent }() if err := r.with(node.With, depth); err != nil { return nil, err } + if expanded, handled, err := r.inlineView(node, depth); handled || err != nil { + return expanded, err + } + clone := *node clone.With = nil clone.From = cloneRelationTables(node.From) @@ -122,15 +165,11 @@ func (r *storedResolver) with(clause *sqlparser.With, depth int) error { return fmt.Errorf("recursive provider CTEs require an iterative execution boundary") } for _, definition := range clause.CTEs { - query, isSelect := definition.Select.(*sqlparser.Select) - if !isSelect { - return fmt.Errorf("CTE %s requires a SELECT boundary", definition.Name.GetRawVal()) - } - prepared, err := r.selectNode(query, depth+1) + prepared, err := r.selectStatement(definition.Select, depth+1) if err != nil { return err } - if err = renameCTEColumns(prepared, definition.Columns); err != nil { + if err = renameCTEColumns(firstSelect(prepared), definition.Columns); err != nil { return err } r.ctes[strings.ToLower(definition.Name.GetRawVal())] = prepared @@ -142,6 +181,9 @@ func renameCTEColumns(node *sqlparser.Select, columns sqlparser.Columns) error { if len(columns) == 0 { return nil } + if node == nil { + return fmt.Errorf("CTE requires a SELECT projection") + } if len(columns) != len(node.SelectExprs) { return fmt.Errorf("CTE column count does not match its projection") } @@ -174,13 +216,11 @@ func (r *storedResolver) table(expr sqlparser.TableExpr, depth int) error { } case *sqlparser.AliasedTableExpr: if sub, isSub := table.Expr.(*sqlparser.Subquery); isSub { - if inner, isSelect := sub.Select.(*sqlparser.Select); isSelect { - prepared, err := r.selectNode(inner, depth+1) - if err != nil { - return err - } - table.Expr = &sqlparser.Subquery{Select: prepared} + prepared, err := r.selectStatement(sub.Select, depth+1) + if err != nil { + return err } + table.Expr = &sqlparser.Subquery{Select: prepared} return nil } name, isName := table.Expr.(sqlparser.TableName) @@ -245,10 +285,7 @@ func (r *storedResolver) register( if len(names) == 0 { return fmt.Errorf("stored relation %s has no declared columns", metadata.GetName()) } - actualName, err := r.system.GetFullyQualifiedTableName(metadata.GetName()) - if err != nil { - return err - } + actualName := r.system.DelimitFullyQualifiedRelationName(metadata.GetName()) // Identifiers come from registered metadata and are quoted; data is never interpolated. //nolint:gosec // G202: metadata identifiers only. statement := "SELECT " + strings.Join(quoted, ",") + " FROM " + actualName @@ -267,7 +304,7 @@ func (r *storedResolver) register( if err != nil { return err } - r.relations[handle] = relation + r.relations[handle] = newOrderedRelation(relation, names) table.Expr = sqlparser.TableName{ QualifierSecond: sqlparser.NewTableIdent(UnstablePrefix + "omni_relations"), Qualifier: sqlparser.NewTableIdent("sources"), Name: sqlparser.NewTableIdent(handleName), @@ -335,11 +372,11 @@ func (r *storedResolver) expandView( if err != nil { return err } - query, isSelect := statement.(*sqlparser.Select) + query, isSelect := statement.(sqlparser.SelectStatement) if !isSelect { return fmt.Errorf("stored view %s requires SELECT", address) } - prepared, err := r.selectNode(query, depth+1) + prepared, err := r.selectStatement(query, depth+1) if err != nil { return err } @@ -354,15 +391,10 @@ func storedSelectFunc( ctx queryContext, node *sqlparser.Select, currentProvider string, ) (func() internaldto.ExecutorOutput, bool) { prepared, stored, err := prepareStoredRelations(ctx, node) - // Preserve established view expansion for queries without a direct provider - // or stored-table read. Those paths also resolve parameterized views and unions. - if !containsProviderRead(node, currentProvider, nil) && len(stored) == 0 { - return nil, false - } if err != nil { return refuse(err), true } - if !containsProviderRead(prepared, currentProvider, stored) && !hasExternalRelations(stored) { + if !containsProviderRead(prepared, currentProvider, stored) && len(stored) == 0 { return nil, false } isDoc, err := fromDocProviders(prepared.From, currentProvider) @@ -374,3 +406,210 @@ func storedSelectFunc( } return docSelectFunc(ctx, prepared, currentProvider, stored) } + +func firstSelect(stmt sqlparser.SelectStatement) *sqlparser.Select { + switch node := stmt.(type) { + case *sqlparser.Select: + return node + case *sqlparser.ParenSelect: + return firstSelect(node.Select) + case *sqlparser.Union: + return firstSelect(node.FirstStatement) + } + return nil +} + +func (r *storedResolver) selectStatement(stmt sqlparser.SelectStatement, depth int) (sqlparser.SelectStatement, error) { + if depth >= derivedRelationMaxDepth { + return nil, fmt.Errorf("stored relation expansion exceeds nesting bound") + } + switch node := stmt.(type) { + case *sqlparser.Select: + return r.selectNode(node, depth) + case *sqlparser.ParenSelect: + clone := *node + var err error + clone.Select, err = r.selectStatement(node.Select, depth+1) + return &clone, err + case *sqlparser.Union: + parent := r.ctes + r.ctes = maps.Clone(parent) + if r.ctes == nil { + r.ctes = map[string]sqlparser.SelectStatement{} + } + defer func() { r.ctes = parent }() + clone := *node + if first := firstSelect(node.FirstStatement); first != nil { + if err := r.with(first.With, depth+1); err != nil { + return nil, err + } + } + var err error + clone.FirstStatement, err = r.selectStatement(node.FirstStatement, depth+1) + if err != nil { + return nil, err + } + clone.UnionSelects = make([]*sqlparser.UnionSelect, len(node.UnionSelects)) + for i, branch := range node.UnionSelects { + child := *branch + child.Statement, err = r.selectStatement(branch.Statement, depth+1) + if err != nil { + return nil, err + } + clone.UnionSelects[i] = &child + } + return &clone, nil + } + return nil, fmt.Errorf("unsupported stored read %T", stmt) +} + +// Only SDK-declared request inputs can replace a stored view's default bindings. +func bindStoredViewInputs(ctx queryContext, view *sqlparser.Select, where *sqlparser.Where, alias string) error { + if len(view.From) != 1 || view.Where == nil { + return nil + } + table, ok := view.From[0].(*sqlparser.AliasedTableExpr) + if !ok { + return nil + } + name, ok := table.Expr.(sqlparser.TableName) + if !ok || name.Qualifier.IsEmpty() { + return nil + } + bundle, ok := docProvider(resolveProvider(name.QualifierSecond.GetRawVal(), ctx.GetCurrentProvider())) + if !ok { + return nil + } + options, err := documentOptions(ctx) + if err != nil { + return err + } + address := UnstablePrefix + bundle + "." + name.Qualifier.GetRawVal() + "." + name.Name.GetRawVal() + declared, err := omnisdk.DescribeTable(registryRoot(ctx), address, options...) + if err != nil { + return err + } + inputs := map[string]bool{} + for _, method := range declared.Methods() { + for _, param := range method.Params() { + inputs[param.Name()] = true + } + } + bindings := viewLiteralBindings(where, alias, alias) + rewritten := sqlparser.Rewrite(view.Where.Expr, func(cursor *sqlparser.Cursor) bool { + compare, isComparison := cursor.Node().(*sqlparser.ComparisonExpr) + if !isComparison || !literalViewBinding(compare) { + return true + } + col, isColumn := compare.Left.(*sqlparser.ColName) + if !isColumn || !inputs[col.Name.GetRawVal()] { + return true + } + if replacement := bindings[col.Name.GetRawVal()]; replacement != nil { + clone := *replacement + clone.Left = col + cursor.Replace(&clone) + return false + } + return true + }, nil) + view.Where.Expr, ok = rewritten.(sqlparser.Expr) + if !ok { + return fmt.Errorf("view binding rewrite requires an expression") + } + return nil +} + +type orderedRelation interface { + omnisdk.Relation + columnOrder() []string +} +type declaredOrderedRelation struct { + omnisdk.Relation + names []string +} + +func newOrderedRelation(relation omnisdk.Relation, names []string) orderedRelation { + return &declaredOrderedRelation{Relation: relation, names: append([]string(nil), names...)} +} +func (r *declaredOrderedRelation) columnOrder() []string { return append([]string(nil), r.names...) } + +func (r *declaredOrderedRelation) Methods() []omnisdk.MethodSignature { + methods := r.Relation.Methods() + result := make([]omnisdk.MethodSignature, len(methods)) + for i, method := range methods { + result[i] = orderedSignature{MethodSignature: method, relation: r} + } + return result +} + +type orderedSignature struct { + omnisdk.MethodSignature + relation orderedRelation +} + +func (s orderedSignature) Columns() []string { return s.relation.columnOrder() } + +// Simple views share the caller's request bindings; aggregations keep their boundary. +func (r *storedResolver) inlineView(node *sqlparser.Select, depth int) (*sqlparser.Select, bool, error) { + if len(node.From) != 1 || r.system == nil { + return nil, false, nil + } + table, isTable := node.From[0].(*sqlparser.AliasedTableExpr) + if !isTable { + return nil, false, nil + } + name, isName := table.Expr.(sqlparser.TableName) + if !isName { + return nil, false, nil + } + address := qualifiedName(name) + view, exists := r.system.GetViewByName(address) + if !exists { + return nil, false, nil + } + if r.active[address] { + return nil, true, fmt.Errorf("stored view cycle at %s", address) + } + parsed, err := sqlparser.Parse(view.GetRawQuery()) + if err != nil { + return nil, true, err + } + projection, isSelect := parsed.(*sqlparser.Select) + if !isSelect || projection.With != nil || needsStaging(projection) { + return nil, false, nil + } + for _, selection := range projection.SelectExprs { + if _, named := selection.(*sqlparser.AliasedExpr); !named { + return nil, false, nil + } + } + maximum := r.ctx.GetRuntimeContext().IndirectDepthMax + if depth+1 > maximum { + return nil, true, fmt.Errorf( + "indirection chain length %d > %d; please do not cite views at too deep a level", depth+1, maximum) + } + cloned, err := sqlparser.Parse(statementSQL(node)) + if err != nil { + return nil, true, err + } + caller, isCaller := cloned.(*sqlparser.Select) + if !isCaller { + return nil, true, fmt.Errorf("view caller must be SELECT") + } + alias := table.As.GetRawVal() + if alias == "" { + alias = name.Name.GetRawVal() + } + if err = bindStoredViewInputs(r.ctx, projection, node.Where, alias); err != nil { + return nil, true, err + } + expanded, err := inlineDocumentView(caller, projection, alias) + if err != nil { + return nil, true, err + } + r.active[address] = true + result, err := r.selectNode(expanded, depth+1) + delete(r.active, address) + return result, true, err +} diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index 2bb1950ef..92e0928bd 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -68,16 +68,13 @@ func (d *standardDocQuery) getBundles() []string { return d.bundles } func fromDocProviders(from sqlparser.TableExprs, currentProvider string) (bool, error) { var docs, others []string var walk func(expr sqlparser.TableExpr) + var walkStatement func(sqlparser.SelectStatement) walk = func(expr sqlparser.TableExpr) { switch node := expr.(type) { case *sqlparser.AliasedTableExpr: if sub, ok := node.Expr.(*sqlparser.Subquery); ok { - if inner, isSelect := sub.Select.(*sqlparser.Select); isSelect { - for _, child := range inner.From { - walk(child) - } - return - } + walkStatement(sub.Select) + return } tableName, isName := node.Expr.(sqlparser.TableName) if isName && tableName.Name.GetRawVal() == "dual" && @@ -108,6 +105,21 @@ func fromDocProviders(from sqlparser.TableExprs, currentProvider string) (bool, others = append(others, sqlparser.String(node)) } } + walkStatement = func(stmt sqlparser.SelectStatement) { + switch node := stmt.(type) { + case *sqlparser.Select: + for _, child := range node.From { + walk(child) + } + case *sqlparser.ParenSelect: + walkStatement(node.Select) + case *sqlparser.Union: + walkStatement(node.FirstStatement) + for _, branch := range node.UnionSelects { + walkStatement(branch.Statement) + } + } + } for _, expr := range from { walk(expr) } diff --git a/internal/stackql/intrinsic/translate_test.go b/internal/stackql/intrinsic/translate_test.go index 5413e07ee..5ba6ab4be 100644 --- a/internal/stackql/intrinsic/translate_test.go +++ b/internal/stackql/intrinsic/translate_test.go @@ -123,17 +123,17 @@ func TestTranslateSelectRefusals(t *testing.T) { func TestFromDocProvidersRefusesMixing(t *testing.T) { withUnstable(t, true) sel := parseSelect(t, "select a.login from stackql_unstable_github.orgs.members a "+ - "inner join github.orgs.members b on a.login = b.login") + "inner join stackql_preview.audit.aws_iam_principals b on a.login = b.login") isDoc, err := fromDocProviders(sel.From, "") if !isDoc { t.Fatalf("expected the query to be claimed") } - want := "stackql_unstable_* relations cannot be combined with github.orgs.members in one query" + want := "stackql_unstable_* relations cannot be combined with stackql_preview.audit.aws_iam_principals in one query" if err == nil || err.Error() != want { t.Fatalf("got %v, want %s", err, want) } - if isDoc, _ := fromDocProviders(parseSelect(t, "select 1 from github.orgs.members").From, ""); isDoc { - t.Fatalf("a registry-only query must not be claimed") + if isDoc, _ := fromDocProviders(parseSelect(t, "select 1 from github.orgs.members").From, ""); !isDoc { + t.Fatalf("a provider query must be claimed by omnisdk") } } @@ -295,7 +295,7 @@ func TestMutationTablesIgnoresImplicitDual(t *testing.T) { } } -func TestDocProviderUnderOmniAll(t *testing.T) { +func TestDocProviderDefaultsToOmni(t *testing.T) { previous := previewCfg t.Cleanup(func() { previewCfg = previous }) for _, tc := range []struct { @@ -304,13 +304,13 @@ func TestDocProviderUnderOmniAll(t *testing.T) { wantBundle string wantDoc bool }{ - {cfg: previewCfgDTO{}, name: "aws", wantDoc: false}, - {cfg: previewCfgDTO{Unstable: true}, name: "aws", wantDoc: false}, + {cfg: previewCfgDTO{}, name: "aws", wantBundle: "aws", wantDoc: true}, + {cfg: previewCfgDTO{Unstable: true}, name: "aws", wantBundle: "aws", wantDoc: true}, {cfg: previewCfgDTO{Unstable: true}, name: "stackql_unstable_aws", wantBundle: "aws", wantDoc: true}, - {cfg: previewCfgDTO{Omni: "all"}, name: "aws", wantBundle: "aws", wantDoc: true}, - {cfg: previewCfgDTO{Omni: "all"}, name: "stackql_unstable_aws", wantBundle: "aws", wantDoc: true}, - {cfg: previewCfgDTO{Omni: "all"}, name: ProviderName, wantDoc: false}, - {cfg: previewCfgDTO{Omni: "all"}, name: "", wantDoc: false}, + {cfg: previewCfgDTO{}, name: "aws", wantBundle: "aws", wantDoc: true}, + {cfg: previewCfgDTO{}, name: "stackql_unstable_aws", wantBundle: "aws", wantDoc: true}, + {cfg: previewCfgDTO{}, name: ProviderName, wantDoc: false}, + {cfg: previewCfgDTO{}, name: "", wantDoc: false}, } { previewCfg = newBackendInput(tc.cfg) bundle, isDoc := docProvider(tc.name) diff --git a/internal/stackql/intrinsic/verification.go b/internal/stackql/intrinsic/verification.go index 2d48e7fa7..fa9d7db5e 100644 --- a/internal/stackql/intrinsic/verification.go +++ b/internal/stackql/intrinsic/verification.go @@ -4,15 +4,15 @@ import ( "fmt" "github.com/stackql-labs/omnisdk/pkg/docparse/stackqldoc" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql/stackql-provider-registry/signing/pqsig" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "gopkg.in/yaml.v2" ) //nolint:nilnil // A nil verifier is the explicit, successful opt-out state. func documentVerifier(runtime dto.RuntimeCtx) (pqsig.Verifier, error) { - var cfg formulation.RegistryConfig + var cfg omnisdk.RegistryConfig if err := yaml.Unmarshal([]byte(runtime.RegistryRaw), &cfg); err != nil { return nil, fmt.Errorf("document verification config: %w", err) } diff --git a/internal/stackql/intrinsic/verification_test.go b/internal/stackql/intrinsic/verification_test.go index 6ff8a5b11..a7a9034ac 100644 --- a/internal/stackql/intrinsic/verification_test.go +++ b/internal/stackql/intrinsic/verification_test.go @@ -6,8 +6,8 @@ import ( "encoding/json" "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql-provider-registry/signing/pqsig" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) func TestDocumentVerificationConfig(t *testing.T) { diff --git a/internal/stackql/jsonpath/jsonpath.go b/internal/stackql/jsonpath/jsonpath.go new file mode 100644 index 000000000..f5c20907a --- /dev/null +++ b/internal/stackql/jsonpath/jsonpath.go @@ -0,0 +1,131 @@ +package jsonpath + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/PaesslerAG/gval" + jp "github.com/PaesslerAG/jsonpath" + + "context" +) + +func Get(path string, value any) (interface{}, error) { + return jp.Get(path, value) +} + +func SplitSearchPath(pathStr string) ([]string, error) { + return splitSearchPath(map[string]interface{}{}, pathStr) +} + +func splitSearchPath(inputMap map[string]interface{}, pathStr string) ([]string, error) { + pathStr = strings.TrimPrefix(pathStr, "$.") + if pathStr == "" { + return []string{}, nil + } + vs := gval.VariableSelector(func(path gval.Evaluables) gval.Evaluable { + return func(c context.Context, v interface{}) (interface{}, error) { + keys, err := path.EvalStrings(c, v) + if err != nil { + return nil, err + } + return keys, nil + } + }) + lang := gval.NewLanguage(append( + []gval.Language{gval.Base()}, + vs, + )...) + rawVal, err := lang.Evaluate( + pathStr, + inputMap, + ) + conformedVal, isStringSlice := rawVal.([]string) + if !isStringSlice { + return nil, fmt.Errorf("cannot accomodate inferred JSON path of type %T", conformedVal) + } + if err != nil { + return nil, err + } + return conformedVal, nil +} + +//nolint:gocognit // Preserve nested JSON path assignment semantics. +func Set(input any, pathStr string, rhs interface{}) error { + if pathStr == "" { + switch input := input.(type) { + case map[string]interface{}: + rhsMap, rhsIsMap := rhs.(map[string]interface{}) + if !rhsIsMap { + return fmt.Errorf("cannot overwrite map with type %T", rhs) + } + for k, _ := range input { + delete(input, k) + } + for k, v := range rhsMap { + input[k] = v + } + return nil + default: + mutatedSer, mutatedSerErr := json.Marshal(rhs) + if mutatedSerErr != nil { + return mutatedSerErr + } + overwriteErr := json.Unmarshal(mutatedSer, input) + if overwriteErr != nil { + return overwriteErr + } + return nil + } + } + switch input := input.(type) { + case map[string]interface{}: + return setMap(input, pathStr, rhs) + default: + ser, serErr := json.Marshal(input) + if serErr != nil { + return serErr + } + m := map[string]interface{}{} + deserErr := json.Unmarshal(ser, &m) + if deserErr != nil { + return deserErr + } + mutateErr := setMap(m, pathStr, rhs) + if mutateErr != nil { + return mutateErr + } + mutatedSer, mutatedSerErr := json.Marshal(m) + if mutatedSerErr != nil { + return mutatedSerErr + } + overwriteErr := json.Unmarshal(mutatedSer, input) + if overwriteErr != nil { + return overwriteErr + } + } + return nil +} + +func setMap(input map[string]interface{}, pathStr string, rhs interface{}) error { + conformedVal, err := splitSearchPath(map[string]interface{}{}, pathStr) + if err != nil { + return err + } + lhs := input + var lhsOk bool + var k string + for i := range conformedVal { + k = conformedVal[i] + if i == len(conformedVal)-1 { + break + } + lhs, lhsOk = lhs[k].(map[string]interface{}) + if !lhsOk { + return fmt.Errorf("disallowed map traversal into type %T on key %s at index %d", lhs[k], k, i) + } + } + lhs[k] = rhs + return nil +} diff --git a/internal/stackql/jsonpath/jsonpath_test.go b/internal/stackql/jsonpath/jsonpath_test.go new file mode 100644 index 000000000..9979f076d --- /dev/null +++ b/internal/stackql/jsonpath/jsonpath_test.go @@ -0,0 +1,111 @@ +package jsonpath_test + +import ( + "testing" + + "gotest.tools/assert" + + "github.com/stackql/stackql/internal/stackql/jsonpath" + + "encoding/json" + + "reflect" +) + +func TestGet(t *testing.T) { + var val interface{} + jsonErr := json.Unmarshal([]byte(`{"a": {"b": "c"}}`), &val) + assert.NilError(t, jsonErr) + val, err := jsonpath.Get("$.a.b", val) + assert.NilError(t, err) + assert.Assert(t, true) + assert.Equal(t, val, "c") +} + +type myPrincipalStruct struct { + Name string `json:"name"` +} + +type myStruct struct { + Principal *myPrincipalStruct `json:"principal"` + Description string `json:"description"` +} + +func TestGetStruct(t *testing.T) { + var val interface{} = myStruct{} + jsonErr := json.Unmarshal([]byte(`{"principal": {"name": "Joe Blogs"}, "description": "Data point 01 about this dude."}`), &val) + assert.NilError(t, jsonErr) + val, err := jsonpath.Get("$.principal.name", val) + assert.NilError(t, err) + assert.Assert(t, true) + assert.Equal(t, val, "Joe Blogs") +} + +func TestSet(t *testing.T) { + initVal := make(map[string]interface{}) + jsonErr := json.Unmarshal([]byte(`{"a": {"b": "c"}}`), &initVal) + assert.NilError(t, jsonErr) + err := jsonpath.Set(initVal, "a.b", float64(22)) + assert.NilError(t, err) + assert.Assert(t, true) + rhs := make(map[string]interface{}) + rhsErr := json.Unmarshal([]byte(`{"a": {"b": 22}}`), &rhs) + assert.NilError(t, rhsErr) + assert.Assert(t, reflect.DeepEqual(initVal, rhs)) +} + +func TestSetStruct(t *testing.T) { + ms := myStruct{} + jsonErr := json.Unmarshal([]byte(`{"principal": {"name": "Joe Blogs"}, "description": "Data point 01 about this dude."}`), &ms) + assert.NilError(t, jsonErr) + err := jsonpath.Set(&ms, "principal.name", "Frank Smith") + assert.NilError(t, err) + assert.Assert(t, true) + rhs := make(map[string]interface{}) + rhsErr := json.Unmarshal([]byte(`{"a": {"b": 22}}`), &rhs) + assert.NilError(t, rhsErr) + assert.Assert(t, reflect.DeepEqual(ms.Principal.Name, "Frank Smith")) +} + +func TestSetStructAll(t *testing.T) { + ms := myStruct{} + var myTotalOverwrite interface{} + jsonErr := json.Unmarshal([]byte(`{"principal": {"name": "Plastic Fantastic"}, "description": "Data point 01 about this dude."}`), &myTotalOverwrite) + assert.NilError(t, jsonErr) + err := jsonpath.Set(&ms, "", myTotalOverwrite) + assert.NilError(t, err) + assert.Assert(t, true) + rhs := make(map[string]interface{}) + rhsErr := json.Unmarshal([]byte(`{"a": {"b": 22}}`), &rhs) + assert.NilError(t, rhsErr) + assert.Assert(t, reflect.DeepEqual(ms.Principal.Name, "Plastic Fantastic")) +} + +func TestSetDollar(t *testing.T) { + initVal := make(map[string]interface{}) + jsonErr := json.Unmarshal([]byte(`{"a": {"b": "c"}}`), &initVal) + assert.NilError(t, jsonErr) + err := jsonpath.Set(initVal, "$.a.b", float64(22)) + assert.NilError(t, err) + assert.Assert(t, true) + rhs := make(map[string]interface{}) + rhsErr := json.Unmarshal([]byte(`{"a": {"b": 22}}`), &rhs) + assert.NilError(t, rhsErr) + assert.Assert(t, reflect.DeepEqual(initVal, rhs)) +} + +func TestSplitPath(t *testing.T) { + val, err := jsonpath.SplitSearchPath("$.a.b") + assert.NilError(t, err) + assert.Assert(t, true) + rhs := []string{"a", "b"} + assert.Assert(t, reflect.DeepEqual(val, rhs)) +} + +func TestSplitPathLong(t *testing.T) { + val, err := jsonpath.SplitSearchPath("$.a.b.c[\"d.e\"]") + assert.NilError(t, err) + assert.Assert(t, true) + rhs := []string{"a", "b", "c", `d.e`} + assert.Assert(t, reflect.DeepEqual(val, rhs)) +} diff --git a/internal/stackql/kstore/kstore.go b/internal/stackql/kstore/kstore.go index 8d97abe66..46d0104e1 100644 --- a/internal/stackql/kstore/kstore.go +++ b/internal/stackql/kstore/kstore.go @@ -4,7 +4,7 @@ import ( "sort" "sync" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) var ( diff --git a/internal/stackql/kstore/kstore_test.go b/internal/stackql/kstore/kstore_test.go index 35c64e3ff..7ede2c33b 100644 --- a/internal/stackql/kstore/kstore_test.go +++ b/internal/stackql/kstore/kstore_test.go @@ -3,8 +3,8 @@ package kstore_test import ( "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/kstore" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) func TestGetKStore_ReturnsNonNil(t *testing.T) { diff --git a/internal/stackql/logging/logging.go b/internal/stackql/logging/logging.go new file mode 100644 index 000000000..c2a9f653d --- /dev/null +++ b/internal/stackql/logging/logging.go @@ -0,0 +1,30 @@ +package logging + +import ( + "io" + + "github.com/sirupsen/logrus" +) + +var ( + Logger *logrus.Logger //nolint:gochecknoglobals // This is convenient as a global variable +) + +func SetLogger(logLevelStr string) { + Logger = logrus.StandardLogger() + logLevel, err := logrus.ParseLevel(logLevelStr) + if err != nil { + Logger.Fatal(err) + } + Logger.SetLevel(logLevel) +} + +func GetLogger() *logrus.Logger { + if Logger != nil { + return Logger + } + tmpLogger := logrus.New() + tmpLogger.SetLevel(logrus.WarnLevel) + tmpLogger.SetOutput(io.Discard) + return tmpLogger +} diff --git a/internal/stackql/logging/logging_test.go b/internal/stackql/logging/logging_test.go new file mode 100644 index 000000000..e7ee54a51 --- /dev/null +++ b/internal/stackql/logging/logging_test.go @@ -0,0 +1,43 @@ +package logging_test + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stackql/stackql/internal/stackql/logging" +) + +func TestSetLogger(t *testing.T) { + // Test with a valid log level + logging.SetLogger("info") + if logging.Logger == nil { + t.Error("Expected Logger to be set, but it's nil.") + } + if logging.Logger.GetLevel() != logrus.InfoLevel { + t.Errorf("Expected log level to be 'info', but it's %v.", logging.Logger.GetLevel()) + } +} + +func TestGetLogger(t *testing.T) { + // Test when Logger is not set (default) + logging.Logger = nil + tmpLogger := logging.GetLogger() + if tmpLogger == nil { + t.Error("Expected Logger to be set, but it's nil.") + } + if tmpLogger.GetLevel() != logrus.WarnLevel { + t.Errorf("Expected log level is warning, but it's %v.", tmpLogger.GetLevel()) + } + + // Test when Logger is set + inputLogger := logrus.New() + inputLogger.SetLevel(logrus.ErrorLevel) + logging.Logger = inputLogger + tmpLogger = logging.GetLogger() + if tmpLogger == nil { + t.Error("Expected Logger to be set, but it's nil.") + } + if tmpLogger.GetLevel() != logrus.ErrorLevel { + t.Errorf("Expected log level is debug, but it's %v.", tmpLogger.GetLevel()) + } +} diff --git a/internal/stackql/mcpbackend/credential_reload.go b/internal/stackql/mcpbackend/credential_reload.go index 2f78bc74c..c1bb52012 100644 --- a/internal/stackql/mcpbackend/credential_reload.go +++ b/internal/stackql/mcpbackend/credential_reload.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/envfile" "github.com/stackql/stackql/internal/stackql/intrinsic" diff --git a/internal/stackql/mcpbackend/credential_reload_test.go b/internal/stackql/mcpbackend/credential_reload_test.go index a3064c340..a8f921f11 100644 --- a/internal/stackql/mcpbackend/credential_reload_test.go +++ b/internal/stackql/mcpbackend/credential_reload_test.go @@ -6,7 +6,8 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) func TestCredentialSource(t *testing.T) { @@ -15,11 +16,11 @@ func TestCredentialSource(t *testing.T) { ac *dto.AuthCtx want string }{ - {"env var key", &dto.AuthCtx{KeyEnvVar: "MY_SECRET"}, "env:MY_SECRET"}, - {"file path", &dto.AuthCtx{KeyFilePath: "/path/key.json"}, "file:/path/key.json"}, - {"file path env var", &dto.AuthCtx{KeyFilePathEnvVar: "KEY_PATH"}, "env:KEY_PATH"}, - {"basic env pair", &dto.AuthCtx{EnvVarUsername: "U", EnvVarPassword: "P"}, "env:U,env:P"}, - {"inline basic", &dto.AuthCtx{Username: "u", Password: "p"}, "inline"}, + {"env var key", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{KeyEnvVar: "MY_SECRET"}}, "env:MY_SECRET"}, + {"file path", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{KeyFilePath: "/path/key.json"}}, "file:/path/key.json"}, + {"file path env var", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{KeyFilePathEnvVar: "KEY_PATH"}}, "env:KEY_PATH"}, + {"basic env pair", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{EnvVarUsername: "U", EnvVarPassword: "P"}}, "env:U,env:P"}, + {"inline basic", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Username: "u", Password: "p"}}, "inline"}, {"nothing", &dto.AuthCtx{}, "none"}, } for _, tc := range cases { @@ -35,7 +36,7 @@ func TestProviderCredentialStatus(t *testing.T) { const varName = "STACKQL_TEST_STATUS_VAR" t.Run("unresolved env var", func(t *testing.T) { os.Unsetenv(varName) //nolint:errcheck // test hygiene - got := providerCredentialStatus("okta", &dto.AuthCtx{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName}) + got := providerCredentialStatus("okta", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName}}) if got.Status != credentialStatusUnresolved { t.Errorf("expected unresolved, got %q (detail %q)", got.Status, got.Detail) } @@ -45,7 +46,7 @@ func TestProviderCredentialStatus(t *testing.T) { }) t.Run("resolved env var", func(t *testing.T) { t.Setenv(varName, "some-secret") - got := providerCredentialStatus("okta", &dto.AuthCtx{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName}) + got := providerCredentialStatus("okta", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName}}) if got.Status != credentialStatusOK { t.Errorf("expected ok, got %q (detail %q)", got.Status, got.Detail) } @@ -54,7 +55,7 @@ func TestProviderCredentialStatus(t *testing.T) { } }) t.Run("uncheckable auth type", func(t *testing.T) { - got := providerCredentialStatus("azure", &dto.AuthCtx{Type: dto.AuthAzureDefaultStr}) + got := providerCredentialStatus("azure", &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: dto.AuthAzureDefaultStr}}) if got.Status != credentialStatusNotChecked { t.Errorf("expected not_checked, got %q", got.Status) } @@ -91,7 +92,7 @@ func TestQueryProviderName(t *testing.T) { func TestCredentialFingerprint(t *testing.T) { const varName = "STACKQL_TEST_FINGERPRINT_VAR" - ac := &dto.AuthCtx{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName} + ac := &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: dto.AuthAPIKeyStr, KeyEnvVar: varName}} t.Setenv(varName, "key-a") before := credentialFingerprint(ac) if before != credentialFingerprint(ac) { @@ -107,7 +108,7 @@ func TestCredentialFingerprint(t *testing.T) { t.Errorf("fingerprint must not embed the value: %q", fp) } } - successor := &dto.AuthCtx{Type: dto.AuthCustomStr, KeyEnvVar: varName, Successor: &dto.AuthCtx{KeyEnvVar: varName + "_2"}} + successor := &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{Type: dto.AuthCustomStr, KeyEnvVar: varName}, Successor: &dto.AuthCtx{AuthConfig: omnisdk.AuthConfig{KeyEnvVar: varName + "_2"}}} t.Setenv(varName+"_2", "s-a") chained := credentialFingerprint(successor) t.Setenv(varName+"_2", "s-b") diff --git a/internal/stackql/mcpbackend/error_classification_test.go b/internal/stackql/mcpbackend/error_classification_test.go index 7e9aa70ee..9c822e7e2 100644 --- a/internal/stackql/mcpbackend/error_classification_test.go +++ b/internal/stackql/mcpbackend/error_classification_test.go @@ -14,15 +14,15 @@ func TestClassifyBackendError_UpstreamStatusVariants(t *testing.T) { retryable bool }{ { - name: "any-sdk detail form 403", + name: "Omni HTTP 403", err: fmt.Errorf( - "upstream provider error: HTTP response error. Status code 403. Detail: 'rate limit exceeded'"), + "httpx: request to https://example.invalid failed: status 403: rate limit exceeded"), status: 403, retryable: false, }, { - name: "any-sdk body form 429", - err: fmt.Errorf(`upstream provider error: Response error. Status code 429. Body: {"message":"slow down"}`), + name: "Omni HTTP 429", + err: fmt.Errorf(`httpx: request to https://example.invalid failed: status 429: {"message":"slow down"}`), status: 429, retryable: true, }, diff --git a/internal/stackql/mcpbackend/mcp_service_stackql.go b/internal/stackql/mcpbackend/mcp_service_stackql.go index 2afaf701f..dea42902c 100644 --- a/internal/stackql/mcpbackend/mcp_service_stackql.go +++ b/internal/stackql/mcpbackend/mcp_service_stackql.go @@ -15,6 +15,7 @@ import ( "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + runtimedto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/pkg/mcp_server" "github.com/stackql/stackql/pkg/mcp_server/dto" @@ -329,10 +330,10 @@ func (b *stackqlMCPService) runPreprocessedQueryJSON(_ context.Context, query st return results, nil } -// upstreamStatusCodeRegex matches the status code fragments any-sdk embeds in +// upstreamStatusCodeRegex matches the status code fragments Omni embeds in // upstream response error strings, eg "HTTP response error. Status code 403.", // "Response error. Status code 429." and "http response status code: 503". -var upstreamStatusCodeRegex = regexp.MustCompile(`(?i)status code:?\s*(\d{3})`) //nolint:gochecknoglobals // compiled once +var upstreamStatusCodeRegex = regexp.MustCompile(`(?i)status(?: code)?:?\s*(\d{3})`) //nolint:gochecknoglobals // compiled once // isRetryableHTTPStatus classifies upstream statuses for MCP clients: // 408 / 429 / 5xx are worth retrying, other 4xx are not (issue #670). @@ -342,12 +343,12 @@ func isRetryableHTTPStatus(status int) bool { status >= http.StatusInternalServerError } -// isCredentialResolutionError matches any-sdk's credential-resolution error -// shapes; these fail before any HTTP request so carry no upstream status. +// Credential resolution fails before any HTTP request and carries no upstream status. func isCredentialResolutionError(err error) bool { msg := err.Error() return strings.Contains(msg, "credentials error") || - strings.Contains(msg, "references empty string") + strings.Contains(msg, "references empty string") || + strings.Contains(msg, "custom credentials cannot be used") } // queryProviderName extracts the provider of the first provider-qualified @@ -440,10 +441,22 @@ func (b *stackqlMCPService) execQuery(query string) (map[string]any, error) { return rv, nil } +// MCP needs upstream failures as structured tool errors. +type strictHandlerContext struct{ handler.HandlerContext } + +func (h strictHandlerContext) GetRuntimeContext() runtimedto.RuntimeCtx { + runtime := h.HandlerContext.GetRuntimeContext() + runtime.StrictUpstreamErrors = true + return runtime +} +func (h strictHandlerContext) Clone() handler.HandlerContext { + return strictHandlerContext{h.HandlerContext.Clone()} +} + func (b *stackqlMCPService) getUpdatedHandlerCtx(query string) (handler.HandlerContext, error) { clonedCtx := b.handlerCtx.Clone() clonedCtx.SetRawQuery(query) - return clonedCtx, nil + return strictHandlerContext{clonedCtx}, nil } func (b *stackqlMCPService) applyQuery(query string) ([]internaldto.ExecutorOutput, bool) { diff --git a/internal/stackql/metadatavisitors/omni_request.go b/internal/stackql/metadatavisitors/omni_request.go new file mode 100644 index 000000000..614678780 --- /dev/null +++ b/internal/stackql/metadatavisitors/omni_request.go @@ -0,0 +1,130 @@ +package metadatavisitors + +import ( + "fmt" + "slices" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/docparse/aot" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/pkg/prettyprint" +) + +// ToOmniInsertStatement renders host SQL from the SDK's declared request shape. +// +//nolint:gocognit // Render parameter and body declarations in one deterministic pass. +func ToOmniInsertStatement( + columns sqlparser.Columns, method omnisdk.MethodSignature, extended, requiredOnly bool, + printer, placeholders *prettyprint.PrettyPrinter, +) (string, error) { + selected := getColsMap(columns) + included := make(map[string]bool) + var names, values, hints []string + appendField := func(name, value, hint string) { + names, values, hints = append(names, printer.RenderColumnName(name)), append(values, value), append(hints, hint) + included[name] = true + } + metadata, _ := method.(omnisdk.RequestMetadata) + inputName := func(name string) string { + if metadata != nil { + return metadata.InputName(name) + } + return name + } + parameters := method.Params() + slices.SortFunc(parameters, func(a, b omnisdk.ParamSignature) int { return strings.Compare(a.Name(), b.Name()) }) + for _, parameter := range parameters { + name := inputName(parameter.Name()) + if parameter.In() == aot.InBody || (columns != nil && !selected[name]) { + continue + } + if columns == nil && (!extended || requiredOnly) && !parameter.Required() { + continue + } + value := printer.RenderTemplateVarAndDelimit(name) + if typed, ok := parameter.(interface{ Type() string }); ok && typed.Type() != "" && typed.Type() != "string" { + value = printer.RenderTemplateVarNoDelimit(name) + } + appendField(name, value, placeholders.RenderTemplateVarPlaceholderNoDelimit(name)) + } + if metadata != nil && metadata.RequestBodySchema() != nil { + body := metadata.RequestBodySchema() + properties := slices.Clone(body.Properties()) + slices.Sort(properties) + for _, property := range properties { + name := "data__" + inputName(property) + if columns != nil && !selected[name] { + continue + } + shape, exists := body.Property(property) + if !exists || !includeRequestProperty(body, shape, property, requiredOnly) { + continue + } + value, err := printer.PrintTemplatedJSON(requestTemplate(shape, ".values."+name, false, 0)) + if err != nil { + return "", err + } + hint, err := placeholders.PrintPlaceholderJSON(requestTemplate(shape, name, true, 0)) + if err != nil { + return "", err + } + appendField(name, value, placeholders.RenderTemplateVarPlaceholderKeyNoDelimit(name)+": "+hint) + } + } + if err := checkAllColumnsPresent(columns, included); err != nil { + return "", err + } + return "<<>>\nINSERT INTO %s(\n" + + strings.Join(names, ",\n") + "\n)\nSELECT\n" + strings.Join(values, ",\n") + "\n;\n", nil +} + +func includeRequestProperty(parent, property aot.Schema, name string, requiredOnly bool) bool { + required := false + if declaration, ok := parent.(aot.RequiredSchema); ok { + required = slices.Contains(declaration.RequiredProperties(), name) + } + if required { + return true + } + if readOnly, ok := property.(aot.ReadOnlySchema); ok && readOnly.ReadOnly() { + return false + } + return !requiredOnly +} + +const maxRequestTemplateDepth = 1 + +func requestTemplate(shape aot.Schema, name string, placeholder bool, depth int) any { + variable := "{{ " + name + " }}" + if placeholder { + variable = "<< " + name + " >>" + } + if shape == nil { + return variable + } + if item, array := shape.Items(); array { + return []any{requestTemplate(item, name+"[0]", placeholder, depth)} + } + if len(shape.Properties()) > 0 { + if depth >= maxRequestTemplateDepth { + return variable + } + object := make(map[string]any) + for _, key := range shape.Properties() { + property, ok := shape.Property(key) + if ok && includeRequestProperty(shape, property, key, false) { + childName := name + "." + key + if placeholder { + childName = key + } + object[key] = requestTemplate(property, childName, placeholder, depth+1) + } + } + return object + } + if typed, ok := shape.(aot.TypedSchema); ok && typed.Type() == "string" && !placeholder { + return fmt.Sprintf("\"%s\"", variable) + } + return variable +} diff --git a/internal/stackql/metadatavisitors/requestvisitors.go b/internal/stackql/metadatavisitors/requestvisitors.go index cba6c9494..b41df8310 100644 --- a/internal/stackql/metadatavisitors/requestvisitors.go +++ b/internal/stackql/metadatavisitors/requestvisitors.go @@ -2,30 +2,11 @@ package metadatavisitors import ( "fmt" - - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - - "sort" "strings" - "github.com/stackql/stackql/pkg/prettyprint" - "github.com/stackql/stackql-parser/go/vt/sqlparser" ) -var ( - _ TemplatedProduct = &standardTemplatedProduct{} -) - -const ( - strType = "string" - objectType = "object" - arrayType = "array" - symLHS = "<< " - symRHS = " >>" -) - type TemplatedProduct interface { GetBody() string GetPlaceholder() string @@ -51,43 +32,6 @@ func (tp *standardTemplatedProduct) GetPlaceholder() string { return tp.placeholder } -type SchemaRequestTemplateVisitor struct { - MaxDepth int - Strategy string - PrettyPrinter *prettyprint.PrettyPrinter - PlaceholderPrettyPrinter *prettyprint.PrettyPrinter - visitedObjects map[string]bool - requiredOnly bool -} - -func NewSchemaRequestTemplateVisitor( - maxDepth int, - strategy string, - prettyPrinter, placeHolderPrettyPrinter *prettyprint.PrettyPrinter, - requiredOnly bool) *SchemaRequestTemplateVisitor { - return &SchemaRequestTemplateVisitor{ - MaxDepth: maxDepth, - Strategy: strategy, - PrettyPrinter: prettyPrinter, - PlaceholderPrettyPrinter: placeHolderPrettyPrinter, - visitedObjects: make(map[string]bool), - requiredOnly: requiredOnly, - } -} - -func (sv *SchemaRequestTemplateVisitor) recordSchemaVisited(schemaKey string) { - sv.visitedObjects[schemaKey] = true -} - -func (sv *SchemaRequestTemplateVisitor) isVisited(schemaKey string, localVisited map[string]bool) bool { - if localVisited != nil { - if localVisited[schemaKey] { - return true - } - } - return sv.visitedObjects[schemaKey] -} - func checkAllColumnsPresent(columns sqlparser.Columns, toInclude map[string]bool) error { var missingColNames []string if columns != nil { @@ -111,479 +55,3 @@ func getColsMap(columns sqlparser.Columns) map[string]bool { } return retVal } - -func isColIncludable(key string, columns sqlparser.Columns, colMap map[string]bool) bool { - colOk := columns == nil - if colOk { - return colOk - } - return colMap[key] -} - -func isRequestBodyParam(paramName string, m formulation.OperationStore) bool { - return m.IsRequestBodyAttributeRenamed(paramName) -} - -//nolint:funlen,gocognit,revive // acceptable -func ToInsertStatement( - columns sqlparser.Columns, - m formulation.OperationStore, - svc formulation.Service, - extended bool, prettyPrinter, - placeHolderPrettyPrinter *prettyprint.PrettyPrinter, - requiredOnly bool, -) (string, error) { - rawParamsToInclude := m.GetNonBodyParameters() - paramsToInclude := make(map[string]formulation.Addressable) - for k, v := range rawParamsToInclude { - paramsToInclude[k] = v - } - if requiredOnly { - rawParamsToInclude = m.GetRequiredNonBodyParameters() - paramsToInclude = make(map[string]formulation.Addressable) - for k, v := range rawParamsToInclude { - paramsToInclude[k] = v - } - } - successfullyIncludedCols := make(map[string]bool) - if !extended { - rawParamsToInclude = m.GetRequiredParameters() - paramsToInclude = make(map[string]formulation.Addressable) - for k, v := range rawParamsToInclude { - paramsToInclude[k] = v - } - for k := range paramsToInclude { - if m.IsRequestBodyAttributeRenamed(k) { - delete(paramsToInclude, k) - } - } - } - if columns != nil { - paramsToInclude = make(map[string]formulation.Addressable) - for _, col := range columns { - cName := col.GetRawVal() - if !isRequestBodyParam(cName, m) { - p, ok := m.GetParameter(cName) - if !ok { - return "", fmt.Errorf("cannot generate insert statement: column '%s' not present", cName) - } - paramsToInclude[cName] = p - successfullyIncludedCols[cName] = true - } - } - } - var includedParamNames []string - for k := range paramsToInclude { - includedParamNames = append(includedParamNames, k) - } - sort.Strings(includedParamNames) - var columnList, exprList, jsonnetPlaholderList []string - for _, s := range includedParamNames { - p, ok := m.GetParameter(s) - if !ok { - return "", fmt.Errorf("'%s'", s) - } - columnList = append(columnList, prettyPrinter.RenderColumnName(s)) - switch p.GetType() { - case strType: - exprList = append(exprList, prettyPrinter.RenderTemplateVarAndDelimit(s)) - default: - exprList = append(exprList, prettyPrinter.RenderTemplateVarNoDelimit(s)) - } - jsonnetPlaholderList = append(jsonnetPlaholderList, placeHolderPrettyPrinter.RenderTemplateVarPlaceholderNoDelimit(s)) - } - - sch, err := m.GetRequestBodySchema() - - if err != nil { - return "", err - } - - if sch == nil { - err = checkAllColumnsPresent(columns, successfullyIncludedCols) - //nolint:lll // acceptable - return "<<>>\nINSERT INTO %s" + "(\n" + strings.Join(columnList, ",\n") + - "\n)\n" + "SELECT\n" + strings.Join(exprList, ",\n") + "\n;\n", err - } - - //nolint:mnd // acceptable - schemaVisitor := NewSchemaRequestTemplateVisitor(2, "", prettyPrinter, placeHolderPrettyPrinter, requiredOnly) - - tVal, _ := schemaVisitor.RetrieveTemplate(sch, m, extended) - - logging.GetLogger().Infoln(fmt.Sprintf("tVal = %v", tVal)) - - colMap := getColsMap(columns) - - if columns != nil { //nolint:nestif // acceptable - for _, c := range columns { - cName := c.GetRawVal() - if !isRequestBodyParam(cName, m) { - continue - } - cNameSuffix, revertErr := m.RevertRequestBodyAttributeRename(cName) - if revertErr != nil { - return "", revertErr - } - if v, ok := tVal[cNameSuffix]; ok { - columnList = append(columnList, prettyPrinter.RenderColumnName(cName)) - exprList = append(exprList, v.GetBody()) - jsonnetPlaholderList = append(jsonnetPlaholderList, v.GetPlaceholder()) - successfullyIncludedCols[cName] = true - } - } - } else { - tValKeysSorted := getSortedKeysTmplMap(tVal) - for _, k := range tValKeysSorted { - v := tVal[k] - if isColIncludable(k, columns, colMap) { - renamedPropertyKey, renamedPropertyErr := m.RenameRequestBodyAttribute(k) - if renamedPropertyErr != nil { - return "", renamedPropertyErr - } - columnList = append(columnList, prettyPrinter.RenderColumnName( - renamedPropertyKey, - )) - exprList = append(exprList, v.GetBody()) - jsonnetPlaholderList = append(jsonnetPlaholderList, v.GetPlaceholder()) - } - } - } - - err = checkAllColumnsPresent(columns, successfullyIncludedCols) - //nolint:lll // acceptable - retVal := "<<>>\nINSERT INTO %s" + "(\n" + strings.Join(columnList, ",\n") + - "\n)\n" + "SELECT\n" + strings.Join(exprList, ",\n") + "\n;\n" - return retVal, err -} - -//nolint:gocognit // acceptable -func (sv *SchemaRequestTemplateVisitor) processSubSchemasMap( - sc formulation.Schema, - method formulation.OperationStore, - properties map[string]formulation.Schema, -) (map[string]TemplatedProduct, error) { - retVal := make(map[string]TemplatedProduct) - for k, ss := range properties { - logging.GetLogger().Infoln(fmt.Sprintf("RetrieveTemplate() k = '%s', ss is nil ? '%t'", k, ss == nil)) - if ss != nil && (k == "" || !sv.isVisited(k, nil)) { //nolint:nestif // acceptable - localSchemaVisitedMap := make(map[string]bool) - localSchemaVisitedMap[k] = true - if !method.IsRequiredRequestBodyProperty(k) && (ss.IsReadOnly() || (sv.requiredOnly && !sc.IsRequired(k))) { - logging.GetLogger().Infoln(fmt.Sprintf("property = '%s' will be skipped", k)) - continue - } - renamedPropertyKey, renamedPropertyErr := method.RenameRequestBodyAttribute(k) - if renamedPropertyErr != nil { - return nil, renamedPropertyErr - } - rv, err := sv.retrieveTemplateVal( - ss, - method.GetService(), - fmt.Sprintf(".values.%s", renamedPropertyKey), - localSchemaVisitedMap) - if err != nil { - return nil, err - } - pl, err := sv.retrieveJsonnetPlaceholderVal( - ss, method.GetService(), renamedPropertyKey, - localSchemaVisitedMap) - if err != nil { - return nil, err - } - var bodyBytes, placeholderBytes string - switch rvt := rv.(type) { - case map[string]interface{}, []interface{}, string: - bodyBytes, err = sv.PrettyPrinter.PrintTemplatedJSON(rvt) - if err != nil { - return nil, err - } - case nil: - continue - default: - return nil, fmt.Errorf("error processing template key '%s' with disallowed type '%T'", k, rvt) - } - switch plt := pl.(type) { - case map[string]interface{}, []interface{}, string: - placeholderBytes, err = sv.PlaceholderPrettyPrinter.PrintPlaceholderJSON(plt) - if err != nil { - return nil, err - } - colName := sv.PlaceholderPrettyPrinter.RenderTemplateVarPlaceholderKeyNoDelimit( - renamedPropertyKey, - ) - placeholderBytes = fmt.Sprintf("%s: %s", colName, placeholderBytes) - case nil: - continue - default: - return nil, fmt.Errorf("error processing placeholder template key '%s' with disallowed type '%T'", k, plt) - } - retVal[k] = NewTemplatedProduct(bodyBytes, placeholderBytes) - } - } - return retVal, nil -} - -func (sv *SchemaRequestTemplateVisitor) RetrieveTemplate( - sc formulation.Schema, - method formulation.OperationStore, - extended bool, //nolint:revive // TODO: review -) (map[string]TemplatedProduct, error) { - retVal := make(map[string]TemplatedProduct) //nolint:ineffassign,staticcheck,wastedassign // TODO: review - sv.recordSchemaVisited(sc.GetName()) - //nolint:gocritic // TODO: review - switch sc.GetType() { - case objectType: - prop, err := sc.GetProperties() - if err != nil { - return nil, err - } - retVal, err = sv.processSubSchemasMap(sc, method, prop) - if len(retVal) != 0 || err != nil { - return retVal, err - } - additionalProperties, additionalProprtiesExist := sc.GetAdditionalProperties() - if additionalProprtiesExist { - additionalProperties.SetKey("k1") - retVal, err = sv.processSubSchemasMap(sc, method, map[string]formulation.Schema{"k1": additionalProperties}) - } - if len(retVal) == 0 { - return nil, nil //nolint:nilnil // TODO: review - } - return retVal, err - } - return nil, fmt.Errorf("templating of request body only supported for object type payload") -} - -//nolint:funlen,gocognit // acceptable -func (sv *SchemaRequestTemplateVisitor) retrieveTemplateVal( - sc formulation.Schema, - svc formulation.Service, //nolint:unparam // TODO: review - objectKey string, - localSchemaVisitedMap map[string]bool, -) (interface{}, error) { - sSplit := strings.Split(objectKey, ".") - oKey := sSplit[len(sSplit)-1] - oPrefix := objectKey //nolint:ineffassign,wastedassign // TODO: review - if len(sSplit) > 1 { - oPrefix = strings.TrimSuffix(objectKey, "."+oKey) - } else { - oPrefix = "" - } - templateValSuffix := oKey - templateValName := oPrefix + "." + templateValSuffix - if oPrefix == "" { - templateValName = templateValSuffix - } - initialLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range localSchemaVisitedMap { - initialLocalSchemaVisitedMap[k] = v - } - switch sc.GetType() { - case objectType: - rv := make(map[string]interface{}) - props, err := sc.GetProperties() - if err != nil { - return nil, err - } - for k, ss := range props { - propertyLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range initialLocalSchemaVisitedMap { - propertyLocalSchemaVisitedMap[k] = v - } - if ss != nil && ((ss.GetType() != arrayType) || !sv.isVisited(ss.GetTitle(), propertyLocalSchemaVisitedMap)) { - if propertyLocalSchemaVisitedMap[ss.GetTitle()] { - return "\"{{ " + templateValName + " }}\"", nil - } - propertyLocalSchemaVisitedMap[ss.GetTitle()] = true - sv, svErr := sv.retrieveTemplateVal(ss, svc, templateValName+"."+k, propertyLocalSchemaVisitedMap) - if svErr != nil { - return nil, svErr - } - if sv != nil { - rv[k] = sv - } - } - } - if len(rv) == 0 { //nolint:nestif // acceptable - additionalProperties, additionalProprtiesExist := sc.GetAdditionalProperties() - if additionalProprtiesExist { - if additionalProperties != nil { - hasProperties := false - propz, pErr := additionalProperties.GetProperties() - if pErr != nil { - return nil, pErr - } - for k, v := range propz { - hasProperties = true - if k == "" { - k = "key" - } - key := fmt.Sprintf("{{ %s[0].%s }}", templateValName, k) - rv[key] = getAdditionalStuff(v, templateValName) - } - if !hasProperties { - key := fmt.Sprintf("{{ %s[0].%s }}", templateValName, "key") - rv[key] = getAdditionalStuff(additionalProperties, templateValName) - } - } - } - } - if len(rv) == 0 { - return nil, nil //nolint:nilnil // TODO: review - } - return rv, nil - case arrayType: - var arr []interface{} - iSch, err := sc.GetItemsSchema() - if err != nil { - return nil, err - } - itemLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range initialLocalSchemaVisitedMap { - itemLocalSchemaVisitedMap[k] = v - } - itemS, err := sv.retrieveTemplateVal(iSch, svc, templateValName+"[0]", itemLocalSchemaVisitedMap) - arr = append(arr, itemS) - if err != nil { - return nil, err - } - return arr, nil - case strType: - return "\"{{ " + templateValName + " }}\"", nil - default: - return "{{ " + templateValName + " }}", nil - } -} - -//nolint:funlen,gocognit // acceptable -func (sv *SchemaRequestTemplateVisitor) retrieveJsonnetPlaceholderVal( - sc formulation.Schema, - svc formulation.Service, //nolint:unparam // TODO: review - objectKey string, - localSchemaVisitedMap map[string]bool, -) (interface{}, error) { - sSplit := strings.Split(objectKey, ".") - oKey := sSplit[len(sSplit)-1] - oPrefix := objectKey //nolint:ineffassign,wastedassign // TODO: review - if len(sSplit) > 1 { - oPrefix = strings.TrimSuffix(objectKey, "."+oKey) - } else { - oPrefix = "" - } - templateValSuffix := oKey - templateValName := oPrefix + "." + templateValSuffix - if oPrefix == "" || true { - templateValName = templateValSuffix - } - initialLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range localSchemaVisitedMap { - initialLocalSchemaVisitedMap[k] = v - } - switch sc.GetType() { - case objectType: - rv := make(map[string]interface{}) - props, err := sc.GetProperties() - if err != nil { - return nil, err - } - for k, ss := range props { - propertyLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range initialLocalSchemaVisitedMap { - propertyLocalSchemaVisitedMap[k] = v - } - if ss != nil && ((ss.GetType() != arrayType) || !sv.isVisited(ss.GetTitle(), propertyLocalSchemaVisitedMap)) { - if propertyLocalSchemaVisitedMap[ss.GetTitle()] { - return symLHS + templateValName + symRHS, nil - } - propertyLocalSchemaVisitedMap[ss.GetTitle()] = true - sv, svErr := sv.retrieveJsonnetPlaceholderVal(ss, svc, templateValName+"."+k, propertyLocalSchemaVisitedMap) - if svErr != nil { - return nil, svErr - } - if sv != nil { - rv[k] = sv - } - } - } - if len(rv) == 0 { //nolint:nestif // acceptable - additionalProperties, additionalProprtiesExist := sc.GetAdditionalProperties() - if additionalProprtiesExist { - hasProperties := false - propz, pErr := additionalProperties.GetProperties() - if pErr != nil { - return nil, pErr - } - for k, v := range propz { - hasProperties = true - if k == "" { - k = "key" - } - key := fmt.Sprintf("<< %s[0].%s >>", templateValName, k) - rv[key] = getAdditionalStuffPlaceholder(v, templateValName) - } - if !hasProperties { - key := fmt.Sprintf("<< %s[0].%s >>", templateValName, "key") - rv[key] = getAdditionalStuffPlaceholder(additionalProperties, templateValName) - } - } - } - if len(rv) == 0 { - return nil, nil //nolint:nilnil // TODO: review - } - return rv, nil - case arrayType: - var arr []interface{} - iSch, err := sc.GetItemsSchema() - if err != nil { - return nil, err - } - itemLocalSchemaVisitedMap := make(map[string]bool) - for k, v := range initialLocalSchemaVisitedMap { - itemLocalSchemaVisitedMap[k] = v - } - itemS, err := sv.retrieveJsonnetPlaceholderVal(iSch, svc, templateValName+"[0]", itemLocalSchemaVisitedMap) - arr = append(arr, itemS) - if err != nil { - return nil, err - } - return arr, nil - case strType: - return symLHS + templateValName + symRHS, nil - default: - return symLHS + templateValName + symRHS, nil - } -} - -func getAdditionalStuff(ss formulation.Schema, templateValName string) string { - valBase := fmt.Sprintf("{{ %s[0].val }}", templateValName) - switch ss.GetType() { - case strType: - return fmt.Sprintf(`"%s"`, valBase) - case "number", "int", "int32", "int64": - return valBase - default: - return valBase - } -} - -func getAdditionalStuffPlaceholder(ss formulation.Schema, templateValName string) string { - valBase := fmt.Sprintf("<< %s[0].val >>", templateValName) - switch ss.GetType() { - case strType: - return valBase - case "number", "int", "int32", "int64": - return valBase - default: - return valBase - } -} - -func getSortedKeysTmplMap(m map[string]TemplatedProduct) []string { - var retVal []string - for k := range m { - retVal = append(retVal, k) - } - sort.Strings(retVal) - return retVal -} diff --git a/internal/stackql/methodselect/methodselect.go b/internal/stackql/methodselect/methodselect.go deleted file mode 100644 index 4bc2d31f0..000000000 --- a/internal/stackql/methodselect/methodselect.go +++ /dev/null @@ -1,98 +0,0 @@ -package methodselect - -import ( - "fmt" - "strings" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/parserutil" -) - -type IMethodSelector interface { - GetMethod(resource formulation.Resource, methodName string) (formulation.StandardOperationStore, error) - - GetMethodForAction( - resource formulation.Resource, - iqlAction string, - parameters parserutil.ColumnKeyedDatastore) (formulation.StandardOperationStore, string, error) -} - -func NewMethodSelector(provider string, version string) (IMethodSelector, error) { - switch provider { //nolint:gocritic // acceptable - default: - return newGoogleMethodSelector(version) - } -} - -func newGoogleMethodSelector(version string) (IMethodSelector, error) { - switch version { //nolint:gocritic // acceptable - default: - return &DefaultMethodSelector{}, nil - } -} - -type DefaultMethodSelector struct { -} - -func (sel *DefaultMethodSelector) GetMethodForAction( - resource formulation.Resource, - iqlAction string, - parameters parserutil.ColumnKeyedDatastore) (formulation.StandardOperationStore, string, error) { - var methodName string - switch strings.ToLower(iqlAction) { - case "select": - methodName = "select" - case "delete": - methodName = "delete" - case "insert": - methodName = "insert" - case "update": - methodName = "update" - case "replace": - methodName = "replace" - default: - return nil, "", fmt.Errorf( - "iql action = '%s' curently not supported, there is no method mapping possible for any resource", - iqlAction) - } - m, err := sel.GetMethodByNameAndParameters(resource, methodName, parameters) - return m, methodName, err -} - -func (sel *DefaultMethodSelector) GetMethod( - resource formulation.Resource, methodName string) (formulation.StandardOperationStore, error) { - return sel.getMethodByName(resource, methodName) -} - -func (sel *DefaultMethodSelector) getMethodByName( - resource formulation.Resource, methodName string) (formulation.StandardOperationStore, error) { - m, err := resource.FindMethod(methodName) - if err != nil { - return nil, fmt.Errorf("no method = '%s' for resource = '%s'", methodName, resource.GetName()) - } - return m, nil -} - -func (sel *DefaultMethodSelector) GetMethodByNameAndParameters( - resource formulation.Resource, methodName string, - parameters parserutil.ColumnKeyedDatastore) (formulation.StandardOperationStore, error) { - stringifiedParams := parameters.GetStringified() - m, remainingParams, ok := resource.GetFirstNamespaceMethodMatchFromSQLVerb(methodName, stringifiedParams) - if !ok { - return nil, fmt.Errorf("no appropriate method = '%s' for resource = '%s'", methodName, resource.GetName()) - } - // TODO: fix this bodge and - // refactor such that: - // - Server selection and variable assignment is AOT and binding - // - Server selection is passed in to `Paramaterize()` - availableServers, availableServersDoExist := m.GetServers() - if availableServersDoExist { - for _, srv := range availableServers { - for k := range srv.Variables { - delete(remainingParams, k) - } - } - } - parameters.DeleteStringMap(remainingParams) - return m, nil -} diff --git a/internal/stackql/methodselect/methodselect_test.go b/internal/stackql/methodselect/methodselect_test.go deleted file mode 100644 index 2111bf6de..000000000 --- a/internal/stackql/methodselect/methodselect_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package methodselect_test - -import ( - "testing" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/methodselect" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stretchr/testify/assert" -) - -func TestGetMethodForAction(t *testing.T) { - sel := &methodselect.DefaultMethodSelector{} - vr := "v0.1.0" - svc, err := formulation.LoadProviderAndServiceFromPaths( - "./testdata/registry/src/aws/"+vr+"/provider.yaml", - "./testdata/registry/src/aws/"+vr+"/services/s3.yaml", - ) - assert.NoError(t, err) - assert.NotNil(t, svc) - - resource, err := svc.GetResource("bucket_acls") - assert.NoError(t, err) - assert.NotNil(t, resource) - - tests := []struct { - name string - action string - parameters parserutil.ColumnKeyedDatastore - wantErr bool - wantMethod string - }{ - {"SELECT action", "SELECT", parserutil.NewParameterMap(), false, "select"}, - {"UNSUPPORTED action", "UNSUPPORTED", parserutil.NewParameterMap(), true, ""}, - {"Mixed case action", "SeLeCt", parserutil.NewParameterMap(), false, "select"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - method, methodName, err := sel.GetMethodForAction(resource, tt.action, tt.parameters) - if tt.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tt.wantMethod, methodName) - assert.NotNil(t, method) - }) - } -} - -func TestGetMethod(t *testing.T) { - sel := &methodselect.DefaultMethodSelector{} - vr := "v0.1.0" - svc, err := formulation.LoadProviderAndServiceFromPaths( - "./testdata/registry/src/aws/"+vr+"/provider.yaml", - "./testdata/registry/src/aws/"+vr+"/services/s3.yaml", - ) - assert.NoError(t, err) - assert.NotNil(t, svc) - - resource, err := svc.GetResource("bucket_acls") - assert.NoError(t, err) - assert.NotNil(t, resource) - - tests := []struct { - name string - methodName string - wantErr bool - }{ - {"Existing method", "get_bucket_acl", false}, - {"Non-existent method", "nonexistent", true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - method, err := sel.GetMethod(resource, tt.methodName) - if tt.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.NotNil(t, method) - }) - } -} - -func TestGetMethodByNameAndParameters(t *testing.T) { - sel := &methodselect.DefaultMethodSelector{} - vr := "v0.1.0" - svc, err := formulation.LoadProviderAndServiceFromPaths( - "./testdata/registry/src/aws/"+vr+"/provider.yaml", - "./testdata/registry/src/aws/"+vr+"/services/s3.yaml", - ) - assert.NoError(t, err) - assert.NotNil(t, svc) - - resource, err := svc.GetResource("bucket_acls") - assert.NoError(t, err) - assert.NotNil(t, resource) - - tests := []struct { - name string - methodName string - parameters parserutil.ColumnKeyedDatastore - wantErr bool - }{ - { - "Existing method with parameters", - "select", - parserutil.NewParameterMap(), - false, - }, - { - "Non-existent method with parameters", - "nonexistent", - parserutil.NewParameterMap(), - true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - method, err := sel.GetMethodByNameAndParameters(resource, tt.methodName, tt.parameters) - if tt.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.NotNil(t, method) - }) - } -} - -func TestNewMethodSelector(t *testing.T) { - tests := []struct { - name string - provider string - version string - wantErr bool - }{ - {"Default provider", "aws", "v0.1.0", false}, - {"Unsupported provider", "unsupported", "v0.1.0", false}, // Defaults to Google - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - selector, err := methodselect.NewMethodSelector(tt.provider, tt.version) - if tt.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.NotNil(t, selector) - }) - } -} diff --git a/internal/stackql/namemangle/name_mangle.go b/internal/stackql/namemangle/name_mangle.go new file mode 100644 index 000000000..298ba0ac6 --- /dev/null +++ b/internal/stackql/namemangle/name_mangle.go @@ -0,0 +1,76 @@ +package name_mangle //nolint:revive,stylecheck // preference + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + _ NameMangler = &standardNameMutator{} + _ NameUnmangler = &standardNameMutator{} +) + +var ( + //nolint:revive // prefer explicit + standardUnmangleRegexp *regexp.Regexp = regexp.MustCompile(`([\S]+)_([0-9]+)$`) +) + +type NameMangler interface { + MangleName(string, ...any) string +} + +type NameUnmangler interface { + UnmangleName(string) (string, error) +} + +func NewViewNameMangler() NameMangler { + return &standardNameMutator{} +} + +func NewViewNameUnmangler() NameUnmangler { + return &standardNameMutator{} +} + +type standardNameMutator struct { +} + +func (v *standardNameMutator) MangleName(base string, varargs ...any) string { + if len(varargs) == 0 { + return base + } + if len(varargs) == 1 { + i, isInt := varargs[0].(int) + if isInt && i == 0 { + return base + } + } + var sb strings.Builder + sb.WriteString(base) + for _, arg := range varargs { + sb.WriteString(fmt.Sprintf("_%v", arg)) + } + return sb.String() +} + +func (v *standardNameMutator) UnmangleName(base string) (string, error) { + matches := standardUnmangleRegexp.FindStringSubmatch(base) + if len(matches) == 0 { + return base, nil + } + if len(matches) != 3 { //nolint:mnd // 3 is the expected length + return "", fmt.Errorf("could not unmangle %s", base) + } + return matches[1], nil +} + +// func (v *standardNameMutator) GetWhereClause(base string) string { +// matches := standardUnmangleRegexp.FindStringSubmatch(base) +// if len(matches) == 0 { +// return base, nil +// } +// if len(matches) != 3 { //nolint:mnd // 3 is the expected length +// return "", fmt.Errorf("could not unmangle %s", base) +// } +// return matches[1], nil +// } diff --git a/internal/stackql/nativedb/select.go b/internal/stackql/nativedb/select.go index 79156ac1a..c99353aa6 100644 --- a/internal/stackql/nativedb/select.go +++ b/internal/stackql/nativedb/select.go @@ -1,6 +1,6 @@ package nativedb -import "github.com/stackql/any-sdk/pkg/streaming" +import "github.com/stackql/stackql/internal/stackql/streaming" type Select interface { GetColumns() []Column diff --git a/internal/stackql/netutils/http_proxy.go b/internal/stackql/netutils/http_proxy.go new file mode 100644 index 000000000..c71561b84 --- /dev/null +++ b/internal/stackql/netutils/http_proxy.go @@ -0,0 +1,116 @@ +package netutils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "os" + "time" +) + +type HTTPContext interface { + GetCABundle() string + GetHTTPProxyHost() string + GetHTTPProxyUser() string + GetHTTPProxyPassword() string + GetHTTPProxyPort() int + GetHTTPProxyScheme() string + GetAPIRequestTimeout() int + GetTLSAllowInsecure() bool +} + +func GetRoundTripper(httpCtx HTTPContext, existingTransport http.RoundTripper) http.RoundTripper { + return getRoundTripper(httpCtx, existingTransport) +} + +func getRoundTripper(httpCtx HTTPContext, existingTransport http.RoundTripper) http.RoundTripper { + var tr *http.Transport + var rt http.RoundTripper + if existingTransport != nil { + switch exTR := existingTransport.(type) { + case *http.Transport: + tr = exTR.Clone() + default: + rt = exTR + } + } else { + tr = &http.Transport{} + } + if httpCtx.GetCABundle() != "" { + rootCAs, err := getCertPool(httpCtx.GetCABundle()) + if err == nil { + config := &tls.Config{ + InsecureSkipVerify: httpCtx.GetTLSAllowInsecure(), //nolint:gosec // intentional, if contraindicated + RootCAs: rootCAs, + } + tr.TLSClientConfig = config + } + } else if httpCtx.GetTLSAllowInsecure() { + config := &tls.Config{ + InsecureSkipVerify: httpCtx.GetTLSAllowInsecure(), //nolint:gosec // intentional, if contraindicated + } + tr.TLSClientConfig = config + } + host := httpCtx.GetHTTPProxyHost() + if host != "" { + if httpCtx.GetHTTPProxyPort() > 0 { + host = fmt.Sprintf("%s:%d", httpCtx.GetHTTPProxyHost(), httpCtx.GetHTTPProxyPort()) + } + var usr *url.Userinfo + if httpCtx.GetHTTPProxyUser() != "" { + usr = url.UserPassword(httpCtx.GetHTTPProxyUser(), httpCtx.GetHTTPProxyPassword()) + } + proxyURL := &url.URL{ + Host: host, + Scheme: httpCtx.GetHTTPProxyScheme(), + User: usr, + } + if tr != nil { + tr.Proxy = http.ProxyURL(proxyURL) + } + } + if tr != nil { + rt = tr + } + return rt +} + +func GetHTTPClient(httpCtx HTTPContext, existingClient *http.Client) *http.Client { + return getHTTPClient(httpCtx, existingClient) +} + +func getHTTPClient(httpCtx HTTPContext, existingClient *http.Client) *http.Client { + var rt http.RoundTripper + if existingClient != nil && existingClient.Transport != nil { + rt = existingClient.Transport + } + return &http.Client{ + Timeout: time.Second * time.Duration(httpCtx.GetAPIRequestTimeout()), + Transport: getRoundTripper(httpCtx, rt), + } +} + +func getCertPool(localCaBundlePath string) (*x509.CertPool, error) { + var lb []byte + var err error + if localCaBundlePath != "" { + lb, err = os.ReadFile(localCaBundlePath) + if err != nil { + return nil, err + } + } + sp, err := x509.SystemCertPool() + if err == nil && sp != nil { + if lb != nil { + sp.AppendCertsFromPEM(lb) + } + return sp, nil + } + vp := x509.NewCertPool() + if lb != nil { + vp.AppendCertsFromPEM(lb) + } + return vp, nil +} diff --git a/internal/stackql/nomenclature/nomenclature.go b/internal/stackql/nomenclature/nomenclature.go new file mode 100644 index 000000000..a0d33e98d --- /dev/null +++ b/internal/stackql/nomenclature/nomenclature.go @@ -0,0 +1,17 @@ +package nomenclature + +import ( + "strings" + + "github.com/stackql/stackql-provider-registry/registry/pkg/nomenclature" +) + +type ProviderDesignation nomenclature.ProviderDesignation + +func ExtractProviderDesignation(providerStr string) (ProviderDesignation, error) { + rv, err := nomenclature.ExtractProviderDesignation(providerStr) + if !strings.ContainsAny(providerStr, ":@") { + rv.Tag = "" + } + return ProviderDesignation(rv), err +} diff --git a/internal/stackql/omnistaging/dialect.go b/internal/stackql/omnistaging/dialect.go index be32141c3..c0b232736 100644 --- a/internal/stackql/omnistaging/dialect.go +++ b/internal/stackql/omnistaging/dialect.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/constants" + "github.com/stackql/stackql/internal/stackql/constants" ) const ( diff --git a/internal/stackql/omnistaging/omnistaging_test.go b/internal/stackql/omnistaging/omnistaging_test.go index 2d1dafee9..79d591ed8 100644 --- a/internal/stackql/omnistaging/omnistaging_test.go +++ b/internal/stackql/omnistaging/omnistaging_test.go @@ -10,9 +10,8 @@ import ( "strconv" "testing" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/stackql/stackql/internal/stackql/sqlengine" . "github.com/stackql/stackql/internal/stackql/omnistaging" //nolint:revive // test reads as package ) @@ -122,10 +121,9 @@ func TestDialectStatements(t *testing.T) { func openSQLite(t *testing.T, path string) *sql.DB { t.Helper() - eng, err := sqlengine.NewSQLEngine(dto.SQLBackendCfg{ - DBEngine: constants.DBEngineSQLite3Embedded, - SQLSystem: constants.SQLDialectSQLite3, - DSN: "file:" + path, + eng, err := sqlengine.NewSQLEngine(sqlengine.Config{ + DBEngine: constants.DBEngineSQLite3Embedded, + DSN: "file:" + path, }, nil) if err != nil { t.Fatalf("cannot open sqlite engine: %v", err) diff --git a/internal/stackql/output/otel_test.go b/internal/stackql/output/otel_test.go index 97e2abd55..d381a0148 100644 --- a/internal/stackql/output/otel_test.go +++ b/internal/stackql/output/otel_test.go @@ -12,9 +12,9 @@ import ( "testing" "time" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/pkg/sink" ) diff --git a/internal/stackql/output/output.go b/internal/stackql/output/output.go index a59c8f752..356c75600 100644 --- a/internal/stackql/output/output.go +++ b/internal/stackql/output/output.go @@ -10,10 +10,10 @@ import ( "strings" "github.com/jackc/pgtype" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/iqlutil" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/psqlwire" "github.com/olekukonko/tablewriter" diff --git a/internal/stackql/output/output_test.go b/internal/stackql/output/output_test.go index 43553ac4d..1c907843b 100644 --- a/internal/stackql/output/output_test.go +++ b/internal/stackql/output/output_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type flushCountingWriter struct { diff --git a/internal/stackql/output/partial_table_test.go b/internal/stackql/output/partial_table_test.go index 9a5e5536f..f4a8e1b44 100644 --- a/internal/stackql/output/partial_table_test.go +++ b/internal/stackql/output/partial_table_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type partialTableStream struct { diff --git a/internal/stackql/parserutil/column_handle.go b/internal/stackql/parserutil/column_handle.go index cf8195369..1802372d4 100644 --- a/internal/stackql/parserutil/column_handle.go +++ b/internal/stackql/parserutil/column_handle.go @@ -1,7 +1,6 @@ package parserutil import ( - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -20,19 +19,3 @@ type ColumnHandle struct { // WithDisplayName re-keys a wire-spelled column reference to the tabulation // display name the backend column carries (snake_case_aliases, any-sdk #131), // keeping the original spelling as the output alias. No-op when the names agree. -func (ch ColumnHandle) WithDisplayName(tabulation formulation.Tabulation) ColumnHandle { - if tabulation == nil || !ch.IsColumn { - return ch - } - for _, tc := range tabulation.GetColumns() { - if tc.GetWireName() == ch.Name && tc.GetName() != ch.Name { - if ch.Alias == "" { - ch.Alias = ch.Name - } - ch.Name = tc.GetName() - ch.DecoratedColumn = "" - return ch - } - } - return ch -} diff --git a/internal/stackql/parserutil/parser_util.go b/internal/stackql/parserutil/parser_util.go index 5e0eaef78..a95b2de30 100644 --- a/internal/stackql/parserutil/parser_util.go +++ b/internal/stackql/parserutil/parser_util.go @@ -5,13 +5,11 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/pkg/astformat" "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/public/formulation" ) const ( @@ -477,20 +475,6 @@ func ExtractSleepDuration(statement *sqlparser.Sleep) (int, error) { return retVal, fmt.Errorf("sleep definition inadequate") } -func CheckColUsagesAgainstTable(colUsages []ColumnUsageMetadata, table formulation.OperationStore) error { - for _, colUsage := range colUsages { - param, ok := table.GetParameter(colUsage.ColName.Name.GetRawVal()) - if ok { - usageErr := CheckSQLParserTypeVsColumn(colUsage, param.ConditionIsValid) - if usageErr != nil { - return usageErr - } - } - logging.GetLogger().Debugln(fmt.Sprintf("colname = %v", colUsage.ColName)) - } - return nil -} - //nolint:gocritic // TODO: review func GetColumnUsageTypes(statement sqlparser.Expr) ([]ColumnUsageMetadata, error) { var colMetaSlice []ColumnUsageMetadata @@ -816,16 +800,6 @@ func getDecoratedColRendition(baseDecoratedColumn, alias string) string { return baseDecoratedColumn } -func CheckSQLParserTypeVsServiceColumn( - colUsage ColumnUsageMetadata) error { - return CheckSQLParserTypeVsColumn(colUsage, formulation.ServiceConditionIsValid) -} - -func CheckSQLParserTypeVsResourceColumn( - colUsage ColumnUsageMetadata) error { - return CheckSQLParserTypeVsColumn(colUsage, formulation.ResourceConditionIsValid) -} - //nolint:mnd // TODO: remove this func CheckSQLParserTypeVsColumn(colUsage ColumnUsageMetadata, verifyCallback func(string, interface{}) bool) error { switch colUsage.ColVal.Type { diff --git a/internal/stackql/persistence/persistence_system.go b/internal/stackql/persistence/persistence_system.go deleted file mode 100644 index 92b18800d..000000000 --- a/internal/stackql/persistence/persistence_system.go +++ /dev/null @@ -1,65 +0,0 @@ -package persistence - -import ( - "github.com/stackql/any-sdk/pkg/name_mangle" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/sql_system" -) - -var ( - _ formulation.PersistenceSystem = &SQLPersistenceSystem{} -) - -type SQLPersistenceSystem struct { - sqlSystem sql_system.SQLSystem - viewNameMangler name_mangle.NameMangler -} - -func NewSQLPersistenceSystem(sqlSystem sql_system.SQLSystem) formulation.PersistenceSystem { - return &SQLPersistenceSystem{ - sqlSystem: sqlSystem, - viewNameMangler: name_mangle.NewViewNameMangler(), - } -} - -func (s *SQLPersistenceSystem) GetSystemName() string { - return s.sqlSystem.GetName() -} - -func (s *SQLPersistenceSystem) HandleExternalTables( - providerName string, externalTables map[string]formulation.SQLExternalTable) error { - for _, tbl := range externalTables { - err := s.sqlSystem.RegisterExternalTable( - providerName, - tbl, - ) - if err != nil { - return err - } - } - return nil -} - -func (s *SQLPersistenceSystem) HandleViewCollection(viewCollection []formulation.View) error { - for i, view := range viewCollection { - viewNameNaive := view.GetNameNaive() - viewName := s.viewNameMangler.MangleName(viewNameNaive, i) - _, viewExists := s.sqlSystem.GetViewByName(viewName) - if !viewExists { - // TODO: resolve any possible data race - err := s.sqlSystem.CreateView(viewName, view.GetDDL(), true, view.GetRequiredParamNames()) - if err != nil { - return err - } - } - } - return nil -} - -func (s *SQLPersistenceSystem) CacheStoreGet(key string) ([]byte, error) { - return s.sqlSystem.GetSQLEngine().CacheStoreGet(key) -} - -func (s *SQLPersistenceSystem) CacheStorePut(key string, value []byte, expiration string, ttl int) error { - return s.sqlSystem.GetSQLEngine().CacheStorePut(key, value, expiration, ttl) -} diff --git a/internal/stackql/planbuilder/entrypoint.go b/internal/stackql/planbuilder/entrypoint.go index 2436b9d32..48d5d4efd 100644 --- a/internal/stackql/planbuilder/entrypoint.go +++ b/internal/stackql/planbuilder/entrypoint.go @@ -3,13 +3,13 @@ package planbuilder import ( "fmt" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/astanalysis/earlyanalysis" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/intrinsic" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parser" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/plan" @@ -107,10 +107,19 @@ func (pb *standardPlanBuilder) BuildPlanFromContext(handlerCtx handler.HandlerCo // An omnisdk data relation streams its rows straight to the output writer, // so it is not backed by a view and analysis would try, and fail, to resolve // it as a registry-backed relation. Plan it here, ahead of that analysis. - if executor, inverse, isStream := intrinsic.GenerateTransactionStreamFunc(handlerCtx, statement); isStream { + executor, inverse, handled := intrinsic.GenerateTransactionStreamFunc(handlerCtx, statement) + if !handled { + executor, handled = intrinsic.GeneratePrimitiveFunc(handlerCtx, statement) + } + if handled { stmtType := sqlparser.ASTToStatementType(statement) qPlan.SetType(stmtType) - qPlan.SetReadOnly(stmtType == sqlparser.StmtSelect) + readOnly := stmtType == sqlparser.StmtSelect + switch statement.(type) { + case *sqlparser.Show, *sqlparser.DescribeTable, *sqlparser.DescribeMethod: + readOnly = true + } + qPlan.SetReadOnly(readOnly) qPlan.SetCacheable(false) qPlan.SetStatement(statement) pGBuilder.getPlanGraphHolder().CreatePrimitiveNode( diff --git a/internal/stackql/planbuilder/plan_builder.go b/internal/stackql/planbuilder/plan_builder.go index f7f6d3610..66dd94eac 100644 --- a/internal/stackql/planbuilder/plan_builder.go +++ b/internal/stackql/planbuilder/plan_builder.go @@ -7,18 +7,14 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/acid/txn_context" - "github.com/stackql/stackql/internal/stackql/astanalysis/routeanalysis" - "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" "github.com/stackql/stackql/internal/stackql/intrinsic" "github.com/stackql/stackql/internal/stackql/iqlerror" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/plan" "github.com/stackql/stackql/internal/stackql/planbuilderinput" @@ -26,8 +22,7 @@ import ( "github.com/stackql/stackql/internal/stackql/primitivebuilder" "github.com/stackql/stackql/internal/stackql/primitivegenerator" "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" + "github.com/stackql/stackql/internal/stackql/registry" "github.com/stackql/stackql/internal/stackql/util" "golang.org/x/mod/semver" @@ -130,8 +125,6 @@ func (pgb *standardPlanGraphBuilder) createInstructionFor(pbi planbuilderinput.P return pgb.handleDescribe(pbi) case *sqlparser.DescribeMethod: return pgb.handleDescribeMethod(pbi) - case *sqlparser.Exec: - return pgb.handleExec(pbi) case *sqlparser.Explain: return pgb.handleExplain(pbi) case *sqlparser.Insert: @@ -211,7 +204,10 @@ func (pgb *standardPlanGraphBuilder) handleExplain(pbi planbuilderinput.PlanBuil if err != nil { return err } - instructionErr := pgb.createInstructionFor(childPbi) + handled, instructionErr := intrinsic.ValidateStatement(pbi.GetHandlerCtx(), explain.Statement) + if !handled { + instructionErr = pgb.createInstructionFor(childPbi) + } explainMessages := []string{} if instructionErr == nil { explainMessages = append(explainMessages, "Execution plan generated successfully") @@ -283,154 +279,17 @@ func (pgb *standardPlanGraphBuilder) pgInternal(pbi planbuilderinput.PlanBuilder } func (pgb *standardPlanGraphBuilder) handleAuth(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetAuth() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Auth", pbi.GetStatement()) - } - primitiveGenerator := pgb.rootPrimitiveGenerator - prov, err := handlerCtx.GetProvider(node.Provider) - if err != nil { - return err + execute, handled := intrinsic.GeneratePrimitiveFunc(pbi.GetHandlerCtx(), pbi.GetStatement()) + if !handled { + return fmt.Errorf("cannot resolve authentication command") } - err = primitiveGenerator.AnalyzeStatement(pbi) - if err != nil { - logging.GetLogger().Debugln(fmt.Sprintf("err = %s", err.Error())) - return err - } - authCtx, authErr := handlerCtx.GetAuthContext(node.Provider) - if authErr != nil { - return authErr - } - pr := primitive.NewMetaDataPrimitive( - prov, - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - authType := strings.ToLower(node.Type) - if node.KeyFilePath != "" { - authCtx.KeyFilePath = node.KeyFilePath - } - if node.KeyEnvVar != "" { - authCtx.KeyEnvVar = node.KeyEnvVar - } - _, err = prov.Auth(authCtx, authType, true) - return internaldto.NewExecutorOutput(nil, nil, nil, nil, err) - }) - pgb.planGraphHolder.GetPrimitiveGraph().CreatePrimitiveNode(pr) + pgb.planGraphHolder.CreatePrimitiveNode(primitive.NewLocalPrimitive( + func(_ primitive.IPrimitiveCtx) internaldto.ExecutorOutput { return execute() })) return nil } func (pgb *standardPlanGraphBuilder) handleAuthRevoke(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - stmt := pbi.GetStatement() - node, ok := stmt.(*sqlparser.AuthRevoke) - if !ok { - return fmt.Errorf("could not cast statement of type '%T' to required AuthRevoke", stmt) - } - primitiveGenerator := pgb.rootPrimitiveGenerator - err := primitiveGenerator.AnalyzeStatement(pbi) - if err != nil { - return err - } - prov, err := handlerCtx.GetProvider(node.Provider) - if err != nil { - return err - } - authCtx, authErr := handlerCtx.GetAuthContext(node.Provider) - if authErr != nil { - return authErr - } - pr := primitive.NewMetaDataPrimitive( - prov, - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return internaldto.NewExecutorOutput(nil, nil, nil, nil, prov.AuthRevoke(authCtx)) - }) - pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil -} - -func (pgb *standardPlanGraphBuilder) handleDescribe(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetDescribeTable() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required DescribeTable", pbi.GetStatement()) - } - primitiveGenerator := pgb.rootPrimitiveGenerator - err := primitiveGenerator.AnalyzeStatement(pbi) - if err != nil { - return err - } - md, err := primitiveGenerator.GetPrimitiveComposer().GetTable(node) - if err != nil { - return err - } - prov, err := md.GetProvider() - if err != nil { - return err - } - var extended bool = strings.TrimSpace(strings.ToUpper(node.Extended)) == "EXTENDED" - var full bool = strings.TrimSpace(strings.ToUpper(node.Full)) == "FULL" - _, isView := md.GetHeirarchyObjects().GetHeirarchyIDs().GetView() - if isView { - stmtCtx, sOk := primitiveGenerator.GetPrimitiveComposer().GetIndirectDescribeSelectCtx() - if !sOk || stmtCtx == nil { - return fmt.Errorf("cannot describe view without context") - } - nonControlColummns := stmtCtx.GetNonControlColumns() - if len(nonControlColummns) < 1 { - return fmt.Errorf("cannot describe view lacking columns") - } - pr := primitive.NewMetaDataPrimitive( - prov, - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return primitivebuilder.NewDescribeViewInstructionExecutor(handlerCtx, md, nonControlColummns, extended, full) - }) - pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil - } - pr := primitive.NewMetaDataPrimitive( - prov, - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return primitivebuilder.NewDescribeTableInstructionExecutor(handlerCtx, md, extended, full) - }) - pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil -} - -func (pgb *standardPlanGraphBuilder) handleDescribeMethod(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetDescribeMethod() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required DescribeMethod", pbi.GetStatement()) - } - providerName := node.Provider.GetRawVal() - serviceName := node.Service.GetRawVal() - resourceName := node.Resource.GetRawVal() - methodName := node.Method.GetRawVal() - if providerName == "" || serviceName == "" || resourceName == "" || methodName == "" { - return fmt.Errorf("DESCRIBE METHOD requires ...") - } - prov, err := handlerCtx.GetProvider(providerName) - if err != nil { - return err - } - runtimeCtx := handlerCtx.GetRuntimeContext() - rsc, err := prov.GetResource(serviceName, resourceName, runtimeCtx) - if err != nil { - return err - } - extended := strings.TrimSpace(strings.ToLower(node.Extended)) == "extended" - pr := primitive.NewMetaDataPrimitive( - prov, - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return primitivebuilder.NewDescribeMethodInstructionExecutor(handlerCtx, rsc, methodName, extended) - }) - pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil + return pgb.handleAuth(pbi) } func (pgb *standardPlanGraphBuilder) arrangePrebuildIndirects( @@ -701,40 +560,12 @@ func (pgb *standardPlanGraphBuilder) handleDelete(pbi planbuilderinput.PlanBuild if err != nil { return err } - insertCtx := primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx() - var selectCtx drm.PreparedStatementCtx - if len(node.SelectExprs) > 0 { - selectCtx = primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx() - } - isPhysicalTable := tbl.IsPhysicalTable() - bldrInput := builder_input.NewBuilderInput( - pgb.planGraphHolder, - handlerCtx, - tbl, - ) - var bldr primitivebuilder.Builder - if !isPhysicalTable { - bldr = primitivebuilder.NewDelete( - pgb.planGraphHolder, - handlerCtx, - insertCtx, - selectCtx, - node, - tbl, - nil, - primitiveGenerator.GetPrimitiveComposer().IsAwait(), - bldrInput, - ) - } else { - tcc := pbi.GetTxnCtrlCtrs() - bldr = primitivebuilder.NewRawNativeExec( - pgb.planGraphHolder, - handlerCtx, - tcc, - handlerCtx.GetQuery(), - bldrInput, - ) + if !tbl.IsPhysicalTable() { + return fmt.Errorf("provider DELETE must be planned through omnisdk") } + bldrInput := builder_input.NewBuilderInput(pgb.planGraphHolder, handlerCtx, tbl) + bldr := primitivebuilder.NewRawNativeExec(pgb.planGraphHolder, handlerCtx, + pbi.GetTxnCtrlCtrs(), handlerCtx.GetQuery(), bldrInput) err = bldr.Build() if err != nil { return err @@ -769,7 +600,7 @@ func (pgb *standardPlanGraphBuilder) handleRegistry(pbi planbuilderinput.PlanBui return pgb.handleRegistryPull(reg, node, pbi) case "list": var colz []string - var provz map[string]formulation.ProviderDescription + var provz map[string]registry.ProviderDescription keys := make(map[string]map[string]interface{}) if node.ProviderId == "" { provz, err = reg.ListAllAvailableProviders() @@ -784,7 +615,7 @@ func (pgb *standardPlanGraphBuilder) handleRegistry(pbi planbuilderinput.PlanBui sort.Strings(dks) for i, k := range dks { v := provz[k] - for _, ver := range v.Versions() { + for _, ver := range v.Versions { keys[strconv.Itoa(i)] = map[string]interface{}{ "provider": k, "version": ver, @@ -801,7 +632,7 @@ func (pgb *standardPlanGraphBuilder) handleRegistry(pbi planbuilderinput.PlanBui for k, v := range provz { keys[strconv.Itoa(i)] = map[string]interface{}{ "provider": k, - "versions": strings.Join(v.Versions(), ", "), + "versions": strings.Join(v.Versions, ", "), } i++ } @@ -821,7 +652,7 @@ func (pgb *standardPlanGraphBuilder) handleRegistry(pbi planbuilderinput.PlanBui // handleRegistryPull is a helper function that pulls a provider version from the registry and persists it locally. func (pgb *standardPlanGraphBuilder) handleRegistryPull( - reg formulation.RegistryAPI, + reg registry.RegistryAPI, node *sqlparser.Registry, pbi planbuilderinput.PlanBuilderInput, ) internaldto.ExecutorOutput { @@ -842,7 +673,7 @@ func (pgb *standardPlanGraphBuilder) handleRegistryPull( } if providerDesc, exists := localProviders[providerID]; exists { // Remove all existing versions - for _, version := range providerDesc.Versions() { + for _, version := range providerDesc.Versions { if err = reg.RemoveProviderVersion(providerID, version); err != nil { return internaldto.NewErroneousExecutorOutput(err) } @@ -1028,11 +859,6 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild if selErr != nil { return selErr } - sr := routeanalysis.NewSelectRoutePass(rowsNode, selPbi, nil) - err = sr.RoutePass() - if err != nil { - return err - } _, selectPrimitiveNode, err = pgb.handleSelect(selPbi) if err != nil { return err @@ -1080,115 +906,13 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild if isPhysicalTable { bldrInput.SetIsTargetPhysicalTable(true) } - return pgb.handleMutationOperation( - handlerCtx, - pbi, - primitiveGenerator, - node, - tbl, - selectPrimitiveNode, - bldrInput, - isAwait, - ) + return primitivebuilder.NewInsertOrUpdate(bldrInput).Build() } pr := primitive.NewGenericPrimitive(nil, nil, nil, primitive_context.NewPrimitiveContext()) pgb.planGraphHolder.CreatePrimitiveNode(pr) return nil } -func (pgb *standardPlanGraphBuilder) handleMutationOperation( - handlerCtx handler.HandlerContext, - pbi planbuilderinput.PlanBuilderInput, - primitiveGenerator primitivegenerator.PrimitiveGenerator, - node sqlparser.SQLNode, - tbl tablemetadata.ExtendedTableMetadata, - selectPrimitiveNode primitivegraph.PrimitiveNode, - bldrInput builder_input.BuilderInput, - isAwait bool, -) error { - var returningExpressions sqlparser.SelectExprs - var inputAction string - var bldr primitivebuilder.Builder - switch n := node.(type) { - case *sqlparser.Insert: - returningExpressions = n.SelectExprs - inputAction = n.Action - case *sqlparser.Update: - returningExpressions = n.SelectExprs - inputAction = n.Action - default: - return fmt.Errorf("unsupported mutation operation of type '%T'", node) - } - //nolint:nestif // acceptable complexity - if len(returningExpressions) > 0 { - // Two cases: - // 1. Synchronous. Equivalent to select. - // 2. Asynchronous. Whole other story. - tableMeta, tableMetaExists := bldrInput.GetTableMetadata() - if !tableMetaExists { - return fmt.Errorf("could not obtain table metadata for node '%s'", inputAction) - } - rc, rcErr := tableinsertioncontainer.NewTableInsertionContainer( - tableMeta, - handlerCtx.GetSQLEngine(), - handlerCtx.GetTxnCounterMgr(), - ) - if rcErr != nil { - return rcErr - } - bldrInput.SetTableInsertionContainer(rc) - bldrInput.SetIsReturning(true) - if !isAwait { - bldr = primitivebuilder.NewSingleAcquireAndSelect( - bldrInput, - primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), - primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), - nil, - ) - } else { - bldrInput.SetIsAwait(true) - bldrInput.SetIsReturning(true) - bldrInput.SetInsertCtx(primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx()) - lhsBldr := primitivebuilder.NewInsertOrUpdate( - bldrInput, - ) - newBldrInput := builder_input.NewBuilderInput( - pgb.planGraphHolder, - handlerCtx, - tbl, - ) - newBldrInput.SetParserNode(node) - newBldrInput.SetAnnotatedAST(pbi.GetAnnotatedAST()) - newBldrInput.SetTxnCtrlCtrs(pbi.GetTxnCtrlCtrs()) - newBldrInput.SetTableInsertionContainer(rc) - newBldrInput.SetDependencyNode(selectPrimitiveNode) - newBldrInput.SetIsAwait(isAwait) - rhsBldr := primitivebuilder.NewSingleSelect( - pgb.planGraphHolder, - handlerCtx, - primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), - []tableinsertioncontainer.TableInsertionContainer{rc}, - nil, - streaming.NewNopMapStream(), - ) - bldr = primitivebuilder.NewDependencySubDAGBuilder( - pgb.planGraphHolder, - []primitivebuilder.Builder{lhsBldr}, - rhsBldr, - ) - } - } else { - bldr = primitivebuilder.NewInsertOrUpdate( - bldrInput, - ) - } - err := bldr.Build() - if err != nil { - return err - } - return nil -} - func (pgb *standardPlanGraphBuilder) handleUpdate(pbi planbuilderinput.PlanBuilderInput) error { handlerCtx := pbi.GetHandlerCtx() node, ok := pbi.GetUpdate() @@ -1240,62 +964,7 @@ func (pgb *standardPlanGraphBuilder) handleUpdate(pbi planbuilderinput.PlanBuild bldrInput.SetTxnCtrlCtrs(pbi.GetTxnCtrlCtrs()) bldrInput.SetIsTargetPhysicalTable(true) } - isAwait := primitiveGenerator.GetPrimitiveComposer().IsAwait() - return pgb.handleMutationOperation( - handlerCtx, - pbi, - primitiveGenerator, - node, - tbl, - selectPrimitiveNode, - bldrInput, - isAwait, - ) - } - pr := primitive.NewGenericPrimitive(nil, nil, nil, primitive_context.NewPrimitiveContext()) - pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil -} - -func (pgb *standardPlanGraphBuilder) handleExec(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetExec() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Exec", pbi.GetStatement()) - } - if !handlerCtx.GetRuntimeContext().TestWithoutAPICalls { //nolint:nestif // acceptable complexity - primitiveGenerator := pgb.rootPrimitiveGenerator - err := primitiveGenerator.AnalyzeStatement(pbi) - if err != nil { - return err - } - // - if primitiveGenerator.IsShowResults() && primitiveGenerator.GetPrimitiveComposer().GetBuilder() != nil { - err = primitiveGenerator.GetPrimitiveComposer().GetBuilder().Build() - if err != nil { - return err - } - return nil - } - tbl, err := primitiveGenerator.GetPrimitiveComposer().GetTable(node) - if err != nil { - return err - } - tcc := pbi.GetTxnCtrlCtrs() - bldr := primitivebuilder.NewExec( - primitiveGenerator.GetPrimitiveComposer().GetGraphHolder(), - handlerCtx, - node, - tbl, - primitiveGenerator.GetPrimitiveComposer().IsAwait(), - primitiveGenerator.IsShowResults(), - tcc, - ) - err = bldr.Build() - if err != nil { - return err - } - return nil + return primitivebuilder.NewInsertOrUpdate(bldrInput).Build() } pr := primitive.NewGenericPrimitive(nil, nil, nil, primitive_context.NewPrimitiveContext()) pgb.planGraphHolder.CreatePrimitiveNode(pr) @@ -1314,9 +983,7 @@ func (pgb *standardPlanGraphBuilder) handleShow(pbi planbuilderinput.PlanBuilder return err } nodeTypeUpper := strings.ToUpper(node.Type) - var tbl tablemetadata.ExtendedTableMetadata - switch nodeTypeUpper { - case "TRANSACTION_ISOLATION_LEVEL": + if nodeTypeUpper == "TRANSACTION_ISOLATION_LEVEL" { builder := primitiveGenerator.GetPrimitiveComposer().GetBuilder() _, isNativeSelect := builder.(*primitivebuilder.NativeSelect) if isNativeSelect { @@ -1324,14 +991,6 @@ func (pgb *standardPlanGraphBuilder) handleShow(pbi planbuilderinput.PlanBuilder return err } return fmt.Errorf("improper usage of 'show transaction isolation level'") - case "INSERT": - tbl, err = primitiveGenerator.GetPrimitiveComposer().GetTable(node) - if err != nil { - return err - } - case "METHODS": - //nolint:wastedassign // TODO: fix this - tbl, err = primitiveGenerator.GetPrimitiveComposer().GetTable(node) //nolint:ineffassign,staticcheck,lll // TODO: fix this } prov := primitiveGenerator.GetPrimitiveComposer().GetProvider() pr := primitive.NewMetaDataPrimitive( @@ -1341,10 +1000,7 @@ func (pgb *standardPlanGraphBuilder) handleShow(pbi planbuilderinput.PlanBuilder return primitivebuilder.NewShowInstructionExecutor( node, prov, - tbl, handlerCtx, - primitiveGenerator.GetPrimitiveComposer().GetCommentDirectives(), - primitiveGenerator.GetPrimitiveComposer().GetTableFilter(), ) }) pgb.planGraphHolder.CreatePrimitiveNode(pr) @@ -1352,17 +1008,7 @@ func (pgb *standardPlanGraphBuilder) handleShow(pbi planbuilderinput.PlanBuilder } func (pgb *standardPlanGraphBuilder) handleSleep(pbi planbuilderinput.PlanBuilderInput) error { - // handlerCtx := pbi.GetHandlerCtx() - _, ok := pbi.GetSleep() - if !ok { - return fmt.Errorf("could not cast statement of type '%T' to required Sleep", pbi.GetStatement()) - } - primitiveGenerator := pgb.rootPrimitiveGenerator - err := primitiveGenerator.AnalyzeStatement(pbi) - if err != nil { - return err - } - return nil + return pgb.handleAuth(pbi) } func (pgb *standardPlanGraphBuilder) handleUse(pbi planbuilderinput.PlanBuilderInput) error { @@ -1417,3 +1063,21 @@ func createErroneousPlan( ) return qPlan, err } + +func (pgb *standardPlanGraphBuilder) handleDescribe(pbi planbuilderinput.PlanBuilderInput) error { + return pgb.handleIntrinsicMetadata(pbi) +} + +func (pgb *standardPlanGraphBuilder) handleDescribeMethod(pbi planbuilderinput.PlanBuilderInput) error { + return pgb.handleIntrinsicMetadata(pbi) +} + +func (pgb *standardPlanGraphBuilder) handleIntrinsicMetadata(pbi planbuilderinput.PlanBuilderInput) error { + execute, handled := intrinsic.GeneratePrimitiveFunc(pbi.GetHandlerCtx(), pbi.GetStatement()) + if !handled { + return fmt.Errorf("cannot resolve metadata for %s", sqlparser.String(pbi.GetStatement())) + } + pgb.planGraphHolder.CreatePrimitiveNode(primitive.NewMetaDataPrimitive(nil, + func(_ primitive.IPrimitiveCtx) internaldto.ExecutorOutput { return execute() })) + return nil +} diff --git a/internal/stackql/planbuilderinput/deprecated.go b/internal/stackql/planbuilderinput/deprecated.go index 9a3970e4b..94753dbaa 100644 --- a/internal/stackql/planbuilderinput/deprecated.go +++ b/internal/stackql/planbuilderinput/deprecated.go @@ -3,7 +3,7 @@ package planbuilderinput import ( "regexp" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/nativedb" "github.com/stackql/stackql/internal/stackql/sqlstream" ) diff --git a/internal/stackql/planbuilderinput/plan_builder_input.go b/internal/stackql/planbuilderinput/plan_builder_input.go index 33270e54f..7fa42db1a 100644 --- a/internal/stackql/planbuilderinput/plan_builder_input.go +++ b/internal/stackql/planbuilderinput/plan_builder_input.go @@ -5,12 +5,9 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/dataflow" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/router" - "github.com/stackql/stackql/internal/stackql/taxonomy" ) type PlanBuilderInput interface { @@ -18,7 +15,6 @@ type PlanBuilderInput interface { Refocus(sqlparser.SQLNode) PlanBuilderInput GetAliasedTables() parserutil.TableAliasMap GetAnnotatedAST() annotatedast.AnnotatedAst - GetAnnotations() (taxonomy.AnnotationCtxMap, bool) GetAuth() (*sqlparser.Auth, bool) GetAuthRevoke() (*sqlparser.AuthRevoke, bool) GetAssignedAliasedColumns() map[sqlparser.TableName]sqlparser.TableExpr @@ -57,8 +53,6 @@ type PlanBuilderInput interface { GetMessages() []string WithMessages(messages []string) PlanBuilderInput - WithParameterRouter(router.ParameterRouter) PlanBuilderInput - WithTableRouteVisitor(tableRouteVisitor router.TableRouteAstVisitor) PlanBuilderInput SetReadOnly(bool) IsReadOnly() bool Next() (PlanBuilderInput, bool) @@ -76,10 +70,6 @@ type StandardPlanBuilderInput struct { tables sqlparser.TableExprs paramsPlaceheld parserutil.ParameterMap tcc internaldto.TxnControlCounters - paramRouter router.ParameterRouter - tableRouteVisitor router.TableRouteAstVisitor - onConditionDataFlows dataflow.Collection - onConditionsToRewrite map[*sqlparser.ComparisonExpr]struct{} tccSetAheadOfTime bool messages []string readOnly bool @@ -205,10 +195,6 @@ func (pbi *StandardPlanBuilderInput) GetMessages() []string { return pbi.messages } -func (pbi *StandardPlanBuilderInput) GetOnConditionsToRewrite() map[*sqlparser.ComparisonExpr]struct{} { - return pbi.onConditionsToRewrite -} - func (pbi *StandardPlanBuilderInput) IsTccSetAheadOfTime() bool { return pbi.tccSetAheadOfTime } @@ -217,39 +203,6 @@ func (pbi *StandardPlanBuilderInput) SetIsTccSetAheadOfTime(tccSetAheadOfTime bo pbi.tccSetAheadOfTime = tccSetAheadOfTime } -func (pbi *StandardPlanBuilderInput) GetOnConditionDataFlows() (dataflow.Collection, bool) { - return pbi.onConditionDataFlows, pbi.onConditionDataFlows != nil -} - -func (pbi *StandardPlanBuilderInput) SetOnConditionsToRewrite( - onConditionsToRewrite map[*sqlparser.ComparisonExpr]struct{}) { - pbi.onConditionsToRewrite = onConditionsToRewrite -} - -func (pbi *StandardPlanBuilderInput) SetOnConditionDataFlows(onConditionDataFlows dataflow.Collection) { - pbi.onConditionDataFlows = onConditionDataFlows -} - -func (pbi *StandardPlanBuilderInput) GetTableMap() (taxonomy.TblMap, bool) { - if pbi.tableRouteVisitor != nil { - return pbi.tableRouteVisitor.GetTableMap(), true - } - return nil, false -} - -func (pbi *StandardPlanBuilderInput) GetAnnotations() (taxonomy.AnnotationCtxMap, bool) { - if pbi.tableRouteVisitor != nil { - return pbi.tableRouteVisitor.GetAnnotations(), true - } - return nil, false -} - -func (pbi *StandardPlanBuilderInput) WithTableRouteVisitor( - tableRouteVisitor router.TableRouteAstVisitor) PlanBuilderInput { - pbi.tableRouteVisitor = tableRouteVisitor - return pbi -} - func (pbi *StandardPlanBuilderInput) GetRawQuery() string { if pbi.rawQuery != "" { return pbi.rawQuery @@ -262,14 +215,6 @@ func (pbi *StandardPlanBuilderInput) SetRawQuery(rawQuery string) { } // router.ParameterRouter. -func (pbi *StandardPlanBuilderInput) GetParameterRouter() (router.ParameterRouter, bool) { - return pbi.paramRouter, true -} - -func (pbi *StandardPlanBuilderInput) WithParameterRouter(paramRouter router.ParameterRouter) PlanBuilderInput { - pbi.paramRouter = paramRouter - return pbi -} func (pbi *StandardPlanBuilderInput) GetAnnotatedAST() annotatedast.AnnotatedAst { return pbi.annotatedAST diff --git a/internal/stackql/presentation/presentation.go b/internal/stackql/presentation/presentation.go index 0f8401e49..17e368525 100644 --- a/internal/stackql/presentation/presentation.go +++ b/internal/stackql/presentation/presentation.go @@ -3,7 +3,7 @@ package presentation import ( "fmt" - "github.com/stackql/any-sdk/pkg/dto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type Driver interface { diff --git a/internal/stackql/primitive/generic_primitive.go b/internal/stackql/primitive/generic_primitive.go index 48d8f6105..17fa6fe13 100644 --- a/internal/stackql/primitive/generic_primitive.go +++ b/internal/stackql/primitive/generic_primitive.go @@ -1,11 +1,11 @@ package primitive import ( - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" + "github.com/stackql/stackql/internal/stackql/logging" ) type GenericPrimitive struct { diff --git a/internal/stackql/primitive/local_primitive.go b/internal/stackql/primitive/local_primitive.go index 43b866afa..eab88c095 100644 --- a/internal/stackql/primitive/local_primitive.go +++ b/internal/stackql/primitive/local_primitive.go @@ -1,10 +1,10 @@ package primitive import ( - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" ) type LocalPrimitive struct { diff --git a/internal/stackql/primitive/primitive.go b/internal/stackql/primitive/primitive.go index d9c77900b..9fe4d4627 100644 --- a/internal/stackql/primitive/primitive.go +++ b/internal/stackql/primitive/primitive.go @@ -3,9 +3,9 @@ package primitive import ( "io" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" ) type IPrimitiveCtx interface { diff --git a/internal/stackql/primitivebuilder/dataflow_graph.go b/internal/stackql/primitivebuilder/dataflow_graph.go deleted file mode 100644 index b70cef9d6..000000000 --- a/internal/stackql/primitivebuilder/dataflow_graph.go +++ /dev/null @@ -1,62 +0,0 @@ -package primitivebuilder - -import ( - "github.com/stackql/any-sdk/public/sqlengine" - "github.com/stackql/stackql/internal/stackql/dataflow" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/util" -) - -type DataflowGraphBuilder struct { - graph primitivegraph.PrimitiveGraphHolder - dataflowGraph dataflow.WeaklyConnectedComponent - handlerCtx handler.HandlerContext - root primitivegraph.PrimitiveNode - sqlEngine sqlengine.SQLEngine -} - -func NewDataflowGraphBuilder( - graph primitivegraph.PrimitiveGraphHolder, - dataflowGraph dataflow.WeaklyConnectedComponent, - txnControlCounters *internaldto.TxnControlCounters, //nolint:revive // future proofing - handlerCtx handler.HandlerContext, - sqlEngine sqlengine.SQLEngine, -) Builder { - return &DataflowGraphBuilder{ - graph: graph, - dataflowGraph: dataflowGraph, - handlerCtx: handlerCtx, - sqlEngine: sqlEngine, - } -} - -func (nb *DataflowGraphBuilder) Build() error { - pr := primitive.NewLocalPrimitive( - //nolint:revive // no big deal - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return util.PrepareResultSet( - internaldto.NewPrepareResultSetPlusRawDTO( - nil, - nil, - nil, - nil, - nil, - internaldto.NewBackendMessages([]string{"nop completed"}), nil, - nb.handlerCtx.GetTypingConfig()), - ) - }, - ) - nb.root = nb.graph.CreatePrimitiveNode(pr) - return nil -} - -func (nb *DataflowGraphBuilder) GetRoot() primitivegraph.PrimitiveNode { - return nb.root -} - -func (nb *DataflowGraphBuilder) GetTail() primitivegraph.PrimitiveNode { - return nb.root -} diff --git a/internal/stackql/primitivebuilder/delete.go b/internal/stackql/primitivebuilder/delete.go deleted file mode 100644 index 0c0808b03..000000000 --- a/internal/stackql/primitivebuilder/delete.go +++ /dev/null @@ -1,134 +0,0 @@ -package primitivebuilder - -import ( - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/asynccompose" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" -) - -type Delete struct { - graph primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - drmCfg drm.Config - root primitivegraph.PrimitiveNode - tail primitivegraph.PrimitiveNode - tbl tablemetadata.ExtendedTableMetadata - node sqlparser.SQLNode - commentDirectives sqlparser.CommentDirectives - isAwait bool - insertCtx drm.PreparedStatementCtx - selectCtx drm.PreparedStatementCtx - bldrInput builder_input.BuilderInput -} - -func NewDelete( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - insertCtx drm.PreparedStatementCtx, - selectCtx drm.PreparedStatementCtx, - node sqlparser.SQLNode, - tbl tablemetadata.ExtendedTableMetadata, - commentDirectives sqlparser.CommentDirectives, - isAwait bool, - bldrInput builder_input.BuilderInput, -) Builder { - return &Delete{ - graph: graph, - handlerCtx: handlerCtx, - drmCfg: handlerCtx.GetDrmConfig(), - bldrInput: bldrInput, - tbl: tbl, - node: node, - commentDirectives: commentDirectives, - isAwait: isAwait, - insertCtx: insertCtx, - selectCtx: selectCtx, - } -} - -func (ss *Delete) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *Delete) GetTail() primitivegraph.PrimitiveNode { - return ss.tail -} - -func (ss *Delete) Build() error { - tbl := ss.tbl - handlerCtx := ss.handlerCtx - prov, err := tbl.GetProvider() - if err != nil { - return err - } - method, methodErr := tbl.GetMethod() - if methodErr != nil { - return methodErr - } - insertContainer, err := tableinsertioncontainer.NewTableInsertionContainer( - tbl, - ss.handlerCtx.GetSQLEngine(), - handlerCtx.GetTxnCounterMgr(), - ) - mvb := execution.NewMonoValentExecutorFactory( - ss.graph, - handlerCtx, - tbl, - ss.insertCtx, - insertContainer, - nil, - streaming.NewNopMapStream(), - !ss.isAwait, - true, - ss.isAwait, - ss.bldrInput, - ) - ex, exErr := mvb.GetExecutor() - if exErr != nil { - return exErr - } - deletePrimitive := primitive.NewGenericPrimitive( - ex, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - if ss.isAwait { - deletePrimitive, err = asynccompose.ComposeAsyncMonitor( - handlerCtx, deletePrimitive, prov, method, nil, false, nil, nil) // isReturning hardcoded to false for now - } - if err != nil { - return err - } - - graph := ss.graph - insertNode := graph.CreatePrimitiveNode(deletePrimitive) - ss.root = insertNode - ss.tail = insertNode - if ss.selectCtx != nil { - selectionBldr := NewSingleSelect( - ss.graph, - handlerCtx, - ss.selectCtx, - []tableinsertioncontainer.TableInsertionContainer{insertContainer}, - nil, - streaming.NewNopMapStream(), - ) - err = selectionBldr.Build() - if err != nil { - return err - } - ss.graph.NewDependency(ss.tail, selectionBldr.GetRoot(), 1.0) - ss.tail = selectionBldr.GetTail() - } - return nil -} diff --git a/internal/stackql/primitivebuilder/dependent_multiple_acquire_and_select.go b/internal/stackql/primitivebuilder/dependent_multiple_acquire_and_select.go deleted file mode 100644 index 5ce619a39..000000000 --- a/internal/stackql/primitivebuilder/dependent_multiple_acquire_and_select.go +++ /dev/null @@ -1,79 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/sql_system" -) - -type DependentMultipleAcquireAndSelect struct { - graph primitivegraph.PrimitiveGraphHolder - acquireBuilders []Builder - selectBuilder Builder - dataflowToEdges map[int][]int - sqlSystem sql_system.SQLSystem - root primitivegraph.PrimitiveNode -} - -func NewDependentMultipleAcquireAndSelect( - graph primitivegraph.PrimitiveGraphHolder, - acquireBuilders []Builder, - selectBuilder Builder, - dataflowToEdges map[int][]int, - sqlSystem sql_system.SQLSystem, -) Builder { - return &DependentMultipleAcquireAndSelect{ - graph: graph, - acquireBuilders: acquireBuilders, - selectBuilder: selectBuilder, - dataflowToEdges: dataflowToEdges, - sqlSystem: sqlSystem, - } -} - -// Cache queries may not have acquire builders. -func (ss *DependentMultipleAcquireAndSelect) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *DependentMultipleAcquireAndSelect) GetTail() primitivegraph.PrimitiveNode { - return ss.selectBuilder.GetTail() -} - -func (ss *DependentMultipleAcquireAndSelect) Build() error { - ss.root = ss.graph.CreatePrimitiveNode( - primitive.NewPassThroughPrimitive( - ss.sqlSystem, - ss.graph.GetTxnControlCounterSlice(), - false, - ), - ) - err := ss.selectBuilder.Build() - if err != nil { - return err - } - ss.graph.NewDependency(ss.root, ss.selectBuilder.GetRoot(), 1.0) - tails := make(map[int]primitivegraph.PrimitiveNode) - for i, acbBld := range ss.acquireBuilders { - err = acbBld.Build() - if err != nil { - return err - } - tail := acbBld.GetTail() - tails[i] = tail - graph := ss.graph - graph.NewDependency(ss.root, acbBld.GetRoot(), 1.0) - graph.NewDependency(tail, ss.selectBuilder.GetRoot(), 1.0) - toEdges := ss.dataflowToEdges[i] - for _, toEdge := range toEdges { - predecessorTail, ok := tails[toEdge] - if !ok { - return fmt.Errorf("unknown predecessor tail") - } - graph.NewDependency(predecessorTail, acbBld.GetRoot(), 1.0) - } - } - return nil -} diff --git a/internal/stackql/primitivebuilder/exec.go b/internal/stackql/primitivebuilder/exec.go deleted file mode 100644 index 4c5940e33..000000000 --- a/internal/stackql/primitivebuilder/exec.go +++ /dev/null @@ -1,188 +0,0 @@ -package primitivebuilder - -import ( - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/asynccompose" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tablemetadata" -) - -type Exec struct { - graph primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - drmCfg drm.Config - root primitivegraph.PrimitiveNode - tbl tablemetadata.ExtendedTableMetadata - isAwait bool - isShowResults bool - tcc internaldto.TxnControlCounters -} - -func NewExec( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - node sqlparser.SQLNode, //nolint:revive // future proofing - tbl tablemetadata.ExtendedTableMetadata, - isAwait bool, - isShowResults bool, - tcc internaldto.TxnControlCounters, -) Builder { - return &Exec{ - graph: graph, - handlerCtx: handlerCtx, - drmCfg: handlerCtx.GetDrmConfig(), - tbl: tbl, - isAwait: isAwait, - isShowResults: isShowResults, - tcc: tcc, - } -} - -func (ss *Exec) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *Exec) GetTail() primitivegraph.PrimitiveNode { - return ss.root -} - -//nolint:gocognit,funlen // probably a headache no matter which way you slice it -func (ss *Exec) Build() error { - handlerCtx := ss.handlerCtx - tbl := ss.tbl - prov, err := tbl.GetProvider() - if err != nil { - return err - } - provider, err := prov.GetProvider() - if err != nil { - return err - } - rtCtx := handlerCtx.GetRuntimeContext() - authCtx, authCtxErr := handlerCtx.GetAuthContext(provider.GetName()) - if authCtxErr != nil { - return authCtxErr - } - outErrFile := handlerCtx.GetOutErrFile() - - m, err := tbl.GetMethod() - if err != nil { - return err - } - svc, err := tbl.GetService() - if err != nil { - return err - } - if ss.isShowResults { - analysisInput := formulation.NewMethodAnalysisInput( - m, - svc, - true, - []formulation.ColumnDescriptor{}, // TODO: get rid of internal refs - false, - ) - analyser := formulation.NewMethodAnalyzer() - methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) - if analysisErr != nil { - return analysisErr - } - methodAnalysisOutput.GetInsertTabulation() - bldrInput := builder_input.NewBuilderInput( - ss.graph, - handlerCtx.Clone(), - tbl, - ) - bldrInput.SetTxnCtrlCtrs(ss.tcc) - deFactoSelectBuilder := NewSingleAcquireAndSelect( - bldrInput, - nil, - nil, - nil, - ) - buildErr := deFactoSelectBuilder.Build() - return buildErr - } - // isNullary := m.IsNullary() - // var target map[string]interface{} - //nolint:revive // no big deal - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - // var columnOrder []string - // keys := make(map[string]map[string]interface{}) - httpArmoury, httpArmouryErr := tbl.GetHTTPArmoury() - if httpArmouryErr != nil { - return internaldto.NewErroneousExecutorOutput(httpArmouryErr) - } - polyHandler := execution.NewStandardPolyHandler( - handlerCtx, - ) - tableName, tableNameErr := tbl.GetTableName() - if tableNameErr != nil { - return internaldto.NewErroneousExecutorOutput(tableNameErr) - } - var singletonBody map[string]interface{} - var rawMessages []string - var readyMessages internaldto.BackendMessages - for _, req := range httpArmoury.GetRequestParams() { - pp := execution.NewProcessorPayload( - req, - execution.NewNilMethodElider(), - provider, - m, - tableName, - rtCtx, - authCtx, - outErrFile, - polyHandler, - "", - nil, - !ss.isShowResults, - true, - ss.isAwait, - false, - !ss.isShowResults, - "", - prov.GetDefaultHTTPClient(), // for testing purposes only - ) - processor := execution.NewProcessor(pp) - processorResponse := processor.Process() - processorErr := processorResponse.GetError() - if processorErr != nil { - return internaldto.NewErroneousExecutorOutput(processorErr) - } - singletonBody = processorResponse.GetSingletonBody() - if len(processorResponse.GetSuccessMessages()) > 0 { - rawMessages = append(rawMessages, processorResponse.GetSuccessMessages()...) - } - } - if len(rawMessages) > 0 { - readyMessages = internaldto.NewBackendMessages(rawMessages) - } - return internaldto.NewExecutorOutput(nil, singletonBody, nil, readyMessages, nil) - } - execPrimitive := primitive.NewGenericPrimitive( - ex, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - if !ss.isAwait { - ss.graph.CreatePrimitiveNode(execPrimitive) - return nil - } - pr, err := asynccompose.ComposeAsyncMonitor( - handlerCtx, execPrimitive, prov, m, - nil, false, nil, nil) // returning hardcoded to false for now - if err != nil { - return err - } - ss.graph.CreatePrimitiveNode(pr) - return nil -} diff --git a/internal/stackql/primitivebuilder/explain.go b/internal/stackql/primitivebuilder/explain.go index 368b3978e..dbee47ef6 100644 --- a/internal/stackql/primitivebuilder/explain.go +++ b/internal/stackql/primitivebuilder/explain.go @@ -3,11 +3,11 @@ package primitivebuilder import ( "time" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/util" ) diff --git a/internal/stackql/primitivebuilder/generic_http_reversal.go b/internal/stackql/primitivebuilder/generic_http_reversal.go deleted file mode 100644 index c03070c33..000000000 --- a/internal/stackql/primitivebuilder/generic_http_reversal.go +++ /dev/null @@ -1,251 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/asynccompose" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/provider" -) - -type genericHTTPReversal struct { - graphHolder primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - drmCfg drm.Config - insertCtx drm.PreparedStatementCtx - root primitivegraph.PrimitiveNode - op formulation.OperationStore - commentDirectives sqlparser.CommentDirectives - isAwait bool - isReturning bool - verb string // may be "insert" or "update" - inputAlias string - isUndo bool - reversalStream formulation.HttpPreparatorStream - prov provider.IProvider -} - -func newGenericHTTPReversal( - builderInput builder_input.BuilderInput, -) (Builder, error) { - handlerCtx, handlerCtxExists := builderInput.GetHandlerContext() - if !handlerCtxExists { - return nil, fmt.Errorf("handler context is required") - } - graphHolder, graphHolderExists := builderInput.GetGraphHolder() - if !graphHolderExists { - return nil, fmt.Errorf("graph holder is required") - } - op, opExists := builderInput.GetOperationStore() - if !opExists { - return nil, fmt.Errorf("operation store is required") - } - prepStream, prepStreamExists := builderInput.GetHTTPPreparatorStream() - if !prepStreamExists { - return nil, fmt.Errorf("preparator stream is required") - } - prov, provExists := builderInput.GetProvider() - if !provExists { - return nil, fmt.Errorf("provider is required") - } - - rv := &genericHTTPReversal{ - prov: prov, - graphHolder: graphHolder, - handlerCtx: handlerCtx, - drmCfg: handlerCtx.GetDrmConfig(), - op: op, - reversalStream: prepStream, - isAwait: builderInput.IsAwait(), - verb: builderInput.GetVerb(), - inputAlias: builderInput.GetInputAlias(), - isUndo: builderInput.IsUndo(), - } - insertCtx, insertCtxExists := builderInput.GetInsertCtx() - if insertCtxExists { - rv.insertCtx = insertCtx - } - return rv, nil -} - -func (gh *genericHTTPReversal) GetRoot() primitivegraph.PrimitiveNode { - return gh.root -} - -func (gh *genericHTTPReversal) GetTail() primitivegraph.PrimitiveNode { - return gh.root -} - -func (gh *genericHTTPReversal) decorateOutput( - op internaldto.ExecutorOutput, tableName string) internaldto.ExecutorOutput { - op.SetUndoLog( - binlog.NewSimpleLogEntry( - nil, - []string{ - fmt.Sprintf("Undo the %s on %s", gh.verb, tableName), - }, - ), - ) - return op -} - -//nolint:funlen,gocognit // TODO: fix this -func (gh *genericHTTPReversal) Build() error { - m := gh.op - prov := gh.prov - provider, providerErr := prov.GetProvider() - if providerErr != nil { - return providerErr - } - handlerCtx := gh.handlerCtx - rtCtx := handlerCtx.GetRuntimeContext() - authCtx, authCtxErr := handlerCtx.GetAuthContext(provider.GetName()) - if authCtxErr != nil { - return authCtxErr - } - outErrFile := handlerCtx.GetOutErrFile() - commentDirectives := gh.commentDirectives - isAwait := gh.isAwait - _, _, responseAnalysisErr := m.GetResponseBodySchemaAndMediaType() - actionPrimitive := primitive.NewGenericPrimitive( - nil, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - tableName := m.GetName() - // target := make(map[string]interface{}) - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - httpPreparator, httpPreparatorExists := gh.reversalStream.Next() - resultSet := internaldto.NewErroneousExecutorOutput(fmt.Errorf("no executions detected")) - var err error - for { - if !httpPreparatorExists { - break - } - httpArmoury, httpErr := httpPreparator.BuildHTTPRequestCtx( - formulation.NewHTTPPreparatorConfig(false), - ) - if httpErr != nil { - return internaldto.NewErroneousExecutorOutput(httpErr) - } - - var nullaryExecutors []func() internaldto.ExecutorOutput - for _, r := range httpArmoury.GetRequestParams() { - req := r - isSkipResponse := responseAnalysisErr != nil - polyHandler := execution.NewStandardPolyHandler( - handlerCtx, - ) - nullaryEx := func() internaldto.ExecutorOutput { - pp := execution.NewProcessorPayload( - req, - execution.NewNilMethodElider(), - provider, - m, - tableName, - rtCtx, - authCtx, - outErrFile, - polyHandler, - "", - nil, - isSkipResponse, - true, - isAwait, - gh.isUndo, - true, - "undo", - prov.GetDefaultHTTPClient(), // for testing purposes only - ) - processor := execution.NewProcessor(pp) - processorResponse := processor.Process() - processorErr := processorResponse.GetError() - singletonBody := processorResponse.GetSingletonBody() - // if processorResponse.IsFailed() && !gh.isAwait { - // processorErr = fmt.Errorf(processorResponse.GetFailedMessage()) - // } - msgs := internaldto.NewBackendMessages(processorResponse.GetSuccessMessages()) - return gh.decorateOutput( - internaldto.NewExecutorOutput( - nil, - singletonBody, - nil, - msgs, - processorErr, - ), - tableName, - ) - } - - nullaryExecutors = append(nullaryExecutors, nullaryEx) - } - if !isAwait { - for _, ei := range nullaryExecutors { - execInstance := ei - aPrioriMessages := resultSet.GetMessages() - resultSet = execInstance() - resultSet.AppendMessages(aPrioriMessages) - if resultSet.GetError() != nil { - return resultSet - } - } - return resultSet - } - for _, eI := range nullaryExecutors { - execInstance := eI - dependentInsertPrimitive := primitive.NewGenericPrimitive( - nil, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - //nolint:revive // no big deal - err = dependentInsertPrimitive.SetExecutor(func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return execInstance() - }) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - execPrim, execErr := asynccompose.ComposeAsyncMonitor( - handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives, gh.isReturning, gh.insertCtx, gh.drmCfg) - if execErr != nil { - return internaldto.NewErroneousExecutorOutput(execErr) - } - resultSet = execPrim.Execute(pc) - if resultSet.GetError() != nil { - return resultSet - } - } - httpPreparator, httpPreparatorExists = gh.reversalStream.Next() - } - return gh.decorateOutput( - resultSet, - tableName, - ) - } - err := actionPrimitive.SetExecutor(ex) - if err != nil { - return err - } - - graphHolder := gh.graphHolder - - // actionNode := graphHolder.CreatePrimitiveNode(actionPrimitive) - - // actionPrimitive.SetInputAlias(gh.inputAlias, gh.dependencyNode.ID()) - - actionNode := graphHolder.CreateInversePrimitiveNode(actionPrimitive) - gh.root = actionNode - return nil -} diff --git a/internal/stackql/primitivebuilder/generic_http_stream_input.go b/internal/stackql/primitivebuilder/generic_http_stream_input.go deleted file mode 100644 index 15ca42695..000000000 --- a/internal/stackql/primitivebuilder/generic_http_stream_input.go +++ /dev/null @@ -1,413 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/client" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/local_template_executor" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/asynccompose" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/util" -) - -type genericHTTPStreamInput struct { - graphHolder primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - drmCfg drm.Config - root primitivegraph.PrimitiveNode - tail primitivegraph.PrimitiveNode - tbl tablemetadata.ExtendedTableMetadata - commentDirectives sqlparser.CommentDirectives - dependencyNode primitivegraph.PrimitiveNode - parserNode sqlparser.SQLNode - isAwait bool - isReturning bool - verb string // may be "insert" or "update" - inputAlias string - isUndo bool - isMutation bool - reversalStream formulation.HttpPreparatorStream - reversalBuilder Builder - rollbackType constants.RollbackType - insertCtx drm.PreparedStatementCtx -} - -func newGenericHTTPStreamInput( - builderInput builder_input.BuilderInput, -) (Builder, error) { - handlerCtx, handlerCtxExists := builderInput.GetHandlerContext() - if !handlerCtxExists { - return nil, fmt.Errorf("handler context is required") - } - graphHolder, graphHolderExists := builderInput.GetGraphHolder() - if !graphHolderExists { - return nil, fmt.Errorf("graph holder is required") - } - tbl, tblExists := builderInput.GetTableMetadata() - if !tblExists { - return nil, fmt.Errorf("table metadata is required") - } - commentDirectives, _ := builderInput.GetCommentDirectives() - dependencyNode, dependencyNodeExists := builderInput.GetDependencyNode() - if !dependencyNodeExists { - return nil, fmt.Errorf("dependency node is required") - } - parserNode, _ := builderInput.GetParserNode() - insertCtx, _ := builderInput.GetInsertCtx() - var isMutation bool - switch parserNode.(type) { - case *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete: - isMutation = true - } - return &genericHTTPStreamInput{ - graphHolder: graphHolder, - handlerCtx: handlerCtx, - drmCfg: handlerCtx.GetDrmConfig(), - tbl: tbl, - commentDirectives: commentDirectives, - dependencyNode: dependencyNode, - isAwait: builderInput.IsAwait(), - isReturning: builderInput.IsReturning(), - verb: builderInput.GetVerb(), - inputAlias: builderInput.GetInputAlias(), - isUndo: builderInput.IsUndo(), - parserNode: parserNode, - reversalStream: formulation.NewHttpPreparatorStream(), - rollbackType: handlerCtx.GetRollbackType(), - insertCtx: insertCtx, - isMutation: isMutation, - }, nil -} - -func (gh *genericHTTPStreamInput) isReverseRequired() bool { - return gh.rollbackType != constants.NopRollback -} - -func (gh *genericHTTPStreamInput) GetRoot() primitivegraph.PrimitiveNode { - return gh.root -} - -func (gh *genericHTTPStreamInput) GetTail() primitivegraph.PrimitiveNode { - if gh.tail != nil { - return gh.tail - } - return gh.root -} - -func (gh *genericHTTPStreamInput) appendReversalData(prep formulation.HTTPPreparator) error { - return gh.reversalStream.Write(prep) -} - -func (gh *genericHTTPStreamInput) decorateOutput( - op internaldto.ExecutorOutput, tableName string) internaldto.ExecutorOutput { - op.SetUndoLog( - binlog.NewSimpleLogEntry( - nil, - []string{ - fmt.Sprintf("Undo the %s on %s", gh.verb, tableName), - }, - ), - ) - return op -} - -func (gh *genericHTTPStreamInput) getInterestingMaps(actionPrimitive primitive.IPrimitive) (mapsAggregatorDTO, error) { - input, inputExists := actionPrimitive.GetInputFromAlias(gh.inputAlias) - if !inputExists { - return nil, fmt.Errorf("input does not exist") - } - inputStream, inputErr := input.ResultToMap() - if inputErr != nil { - return nil, inputErr - } - rr, rrErr := inputStream.Read() - if rrErr != nil { - return nil, rrErr - } - inputMap, inputErr := rr.GetMap() - if inputErr != nil { - return nil, inputErr - } - paramMap, paramErr := util.ExtractSQLNodeParams(gh.parserNode, inputMap) - if paramErr != nil { - return nil, paramErr - } - return newMapsAggregatorDTO(paramMap, inputMap), nil -} - -//nolint:funlen,gocognit,gocyclo,cyclop // TODO: fix this -func (gh *genericHTTPStreamInput) Build() error { - tbl := gh.tbl - handlerCtx := gh.handlerCtx - commentDirectives := gh.commentDirectives - isAwait := gh.isAwait - prov, err := tbl.GetProvider() - if err != nil { - return err - } - provider, providerErr := prov.GetProvider() - if providerErr != nil { - return providerErr - } - rtCtx := handlerCtx.GetRuntimeContext() - authCtx, authCtxErr := handlerCtx.GetAuthContext(provider.GetName()) - if authCtxErr != nil { - return authCtxErr - } - outErrFile := handlerCtx.GetOutErrFile() - svc, err := tbl.GetService() - if err != nil { - return err - } - m, err := tbl.GetMethod() - if err != nil { - return err - } - inverse, inverseExists := m.GetInverse() - if !inverseExists && gh.isReverseRequired() { - return fmt.Errorf("inverse is required") - } - if inverseExists { - inverseOpStore, inverseOpStoreExists := inverse.GetOperationStore() - if !inverseOpStoreExists { - return fmt.Errorf("inverse operation store is required") - } - logging.GetLogger().Debugf("inverse = %v", inverse) - var reversalBuildInitErr error - reverseInput := builder_input.NewBuilderInput( - gh.graphHolder, - handlerCtx, - nil, // tbl, - ) - reverseInput.SetHTTPPreparatorStream(gh.reversalStream) - reverseInput.SetOperationStore(inverseOpStore) - reverseInput.SetProvider(prov) - reverseInput.SetInsertCtx(gh.insertCtx) - gh.reversalBuilder, reversalBuildInitErr = newGenericHTTPReversal(reverseInput) - if reversalBuildInitErr != nil { - return reversalBuildInitErr - } - buildErr := gh.reversalBuilder.Build() - if buildErr != nil { - return buildErr - } - // inverseInput := - // inverseBuilder := - } - responseSchema, _, responseAnalysisErr := tbl.GetResponseSchemaAndMediaType() - actionPrimitive := primitive.NewGenericPrimitive( - nil, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - // reversalStream := streaming.NewStandardMapStream() - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - pr, prErr := prov.GetProvider() - if prErr != nil { - return internaldto.NewErroneousExecutorOutput(prErr) - } - protocolType, protocolTypeErr := pr.GetProtocolType() - if protocolTypeErr != nil { - return internaldto.NewErroneousExecutorOutput(protocolTypeErr) - } - interestingMaps, mapsErr := gh.getInterestingMaps(actionPrimitive) - if mapsErr != nil { - return internaldto.NewErroneousExecutorOutput(mapsErr) - } - paramMap := interestingMaps.getParameterMap() - params := paramMap[0] - //nolint:exhaustive // no big deal - switch protocolType { - case client.LocalTemplated: - inlines := m.GetInline() - if len(inlines) == 0 { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("no inlines found")) - } - executor := local_template_executor.NewLocalTemplateExecutor( - inlines[0], - inlines[1:], - nil, - ) - resp, exErr := executor.Execute( - map[string]any{"parameters": params}, - ) - if exErr != nil { - return internaldto.NewErroneousExecutorOutput(exErr) - } - var backendMessages []string - stdOut, stdOutExists := resp.GetStdOut() - if stdOutExists { - backendMessages = append(backendMessages, stdOut.String()) - } - stdErr, stdErrExists := resp.GetStdErr() - if stdErrExists { - backendMessages = append(backendMessages, stdErr.String()) - } - backendMessages = append(backendMessages, "OK") - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - internaldto.NewBackendMessages(backendMessages), - nil, - ) - case client.HTTP: - httpPreparator := formulation.NewHTTPPreparator( - pr, - svc, - m, - paramMap, - nil, - nil, - logging.GetLogger(), - ) - httpArmoury, httpErr := httpPreparator.BuildHTTPRequestCtx( - formulation.NewHTTPPreparatorConfig(false), - ) - if httpErr != nil { - return internaldto.NewErroneousExecutorOutput(httpErr) - } - - tableName, _ := tbl.GetTableName() - - // var reversalParameters []map[string]interface{} - // TODO: Implement reversal operations: - // - depending upon reversal operation, collect sequence of HTTP operations. - // - for each HTTP operation, collect context and store in *some object* - // then attach to reversal graph for later retrieval and execution - var nullaryExecutors []func() internaldto.ExecutorOutput - for _, r := range httpArmoury.GetRequestParams() { - req := r - isSkipResponse := responseAnalysisErr != nil || responseSchema == nil - polyHandler := execution.NewStandardPolyHandler( - handlerCtx, - ) - nullaryEx := func() internaldto.ExecutorOutput { - pp := execution.NewProcessorPayload( - req, - execution.NewNilMethodElider(), - provider, - m, - tableName, - rtCtx, - authCtx, - outErrFile, - polyHandler, - "", - nil, - isSkipResponse, - true, - isAwait, - gh.isUndo, - gh.isMutation, - "", - prov.GetDefaultHTTPClient(), // for testing purposes only - ) - processor := execution.NewProcessor(pp) - processorResponse := processor.Process() - processorErr := processorResponse.GetError() - singletonBody := processorResponse.GetSingletonBody() - reversalStrem := processorResponse.GetReversalStream() - for { - rev, isRevExistent := reversalStrem.Next() - if !isRevExistent { - break - } - revErr := gh.appendReversalData(rev) - if revErr != nil { - return internaldto.NewErroneousExecutorOutput(revErr) - } - } - // if processorResponse.IsFailed() && !gh.isAwait { - // processorErr = fmt.Errorf(processorResponse.GetFailedMessage()) - // } - return gh.decorateOutput( - internaldto.NewExecutorOutput( - nil, - singletonBody, - nil, - internaldto.NewBackendMessages(processorResponse.GetSuccessMessages()), - processorErr, - ), - tableName, - ) - } - - nullaryExecutors = append(nullaryExecutors, nullaryEx) - } - resultSet := internaldto.NewErroneousExecutorOutput(fmt.Errorf("no executions detected")) - if !isAwait { - for _, ei := range nullaryExecutors { - execInstance := ei - aPrioriMessages := resultSet.GetMessages() - resultSet = execInstance() - resultSet.AppendMessages(aPrioriMessages) - if resultSet.GetError() != nil { - return resultSet - } - } - return resultSet - } - for _, eI := range nullaryExecutors { - execInstance := eI - dependentInsertPrimitive := primitive.NewGenericPrimitive( - nil, - nil, - nil, - primitive_context.NewPrimitiveContext(), - ) - //nolint:revive // no big deal - err = dependentInsertPrimitive.SetExecutor(func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - return execInstance() - }) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - execPrim, execErr := asynccompose.ComposeAsyncMonitor( - handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives, gh.isReturning, gh.insertCtx, gh.drmCfg) - if execErr != nil { - return internaldto.NewErroneousExecutorOutput(execErr) - } - resultSet = execPrim.Execute(pc) - if resultSet.GetError() != nil { - return resultSet - } - } - return gh.decorateOutput( - resultSet, - tableName, - ) - default: - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("unsupported protocol type: %v", protocolType)) - } - } - err = actionPrimitive.SetExecutor(ex) - if err != nil { - return err - } - - graphHolder := gh.graphHolder - - //nolint:errcheck // TODO: fix this - actionPrimitive.SetInputAlias(gh.inputAlias, gh.dependencyNode.ID()) - actionNode := graphHolder.CreatePrimitiveNode(actionPrimitive) - graphHolder.NewDependency(gh.dependencyNode, actionNode, 1.0) - gh.root = gh.dependencyNode - gh.tail = actionNode - - return nil -} diff --git a/internal/stackql/primitivebuilder/graphql_single_select_acquire.go b/internal/stackql/primitivebuilder/graphql_single_select_acquire.go deleted file mode 100644 index 307a663d9..000000000 --- a/internal/stackql/primitivebuilder/graphql_single_select_acquire.go +++ /dev/null @@ -1,280 +0,0 @@ -package primitivebuilder - -import ( - "errors" - "fmt" - "io" - - "github.com/stackql/any-sdk/public/formulation" - - "github.com/stackql/any-sdk/pkg/graphql" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" -) - -// GraphQLSingleSelectAcquire implements the Builder interface -// and represents the action of acquiring data from an endpoint -// and then persisting that data into a table. -// This data would then subsequently be queried by later execution phases. -type GraphQLSingleSelectAcquire struct { - graph primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - tableMeta tablemetadata.ExtendedTableMetadata - drmCfg drm.Config - insertPreparedStatementCtx drm.PreparedStatementCtx - insertionContainer tableinsertioncontainer.TableInsertionContainer - txnCtrlCtr internaldto.TxnControlCounters - rowSort func(map[string]map[string]interface{}) []string - root primitivegraph.PrimitiveNode - stream streaming.MapStream - bldrInput builder_input.BuilderInput -} - -func newGraphQLSingleSelectAcquire( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - tableMeta tablemetadata.ExtendedTableMetadata, - insertCtx drm.PreparedStatementCtx, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, - bldrInput builder_input.BuilderInput, -) Builder { - var tcc internaldto.TxnControlCounters - if insertCtx != nil { - tcc = insertCtx.GetGCCtrlCtrs() - } - if stream == nil { - stream = streaming.NewNopMapStream() - } - return &GraphQLSingleSelectAcquire{ - graph: graph, - handlerCtx: handlerCtx, - tableMeta: tableMeta, - rowSort: rowSort, - drmCfg: handlerCtx.GetDrmConfig(), - insertPreparedStatementCtx: insertCtx, - insertionContainer: insertionContainer, - txnCtrlCtr: tcc, - stream: stream, - bldrInput: bldrInput, - } -} - -func (ss *GraphQLSingleSelectAcquire) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *GraphQLSingleSelectAcquire) GetTail() primitivegraph.PrimitiveNode { - return ss.root -} - -//nolint:govet,funlen,gocognit,revive,errcheck,stylecheck // TODO: fix -func (ss *GraphQLSingleSelectAcquire) Build() error { - prov, err := ss.tableMeta.GetProvider() - if err != nil { - return err - } - provider, err := prov.GetProvider() - if err != nil { - return err - } - authCtx, err := ss.handlerCtx.GetAuthContext(provider.GetName()) - if err != nil { - return err - } - httpArmoury, err := ss.tableMeta.GetHTTPArmoury() - if err != nil { - return err - } - gql, ok := ss.tableMeta.GetGraphQL() - if !ok { - return fmt.Errorf("could not build graphql exection for table") - } - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - currentTcc := ss.insertPreparedStatementCtx.GetGCCtrlCtrs().Clone() - ss.graph.AddTxnControlCounters(currentTcc) - - for _, reqCtx := range httpArmoury.GetRequestParams() { - req := reqCtx.GetRequest() - // Emit the GraphQL wire request + raw response when --http.log.enabled is set - // (alpha08 ContextWithHTTPLogger), mirroring the REST acquire path. - if ss.handlerCtx.GetRuntimeContext().HTTPLogEnabled { - req = req.WithContext( - graphql.ContextWithHTTPLogger(req.Context(), ss.handlerCtx.GetOutErrFile())) - } - housekeepingDone := false - cc := formulation.NewAnySdkClientConfigurator( - ss.handlerCtx.GetRuntimeContext(), prov.GetProviderString(), - prov.GetDefaultHTTPClient()) - client, err := cc.Auth(authCtx, authCtx.Type, false) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - paramMap, err := reqCtx.GetParameters().ToFlatMap() - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - // Carry out the push-down plan: a SQL LIMIT N resolved during the analysis phase - // becomes the GraphQL `limit` query variable, so a provider query template - // referencing {{ .limit }} can bound the page size. - if limit, hasLimit := ss.bldrInput.GetPushdownLimit(); hasLimit { - paramMap["limit"] = limit - } - cursorJsonPath, ok := gql.GetCursorJSONPath() - if !ok { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("cannot perform graphql action without cursor json path")) - } - responseJsonPath, ok := gql.GetResponseJSONPath() - if !ok { - return internaldto.NewErroneousExecutorOutput( - fmt.Errorf("cannot perform graphql action without response json path"), - ) - } - tableName, err := ss.tableMeta.GetTableName() - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - reqEncoding := reqCtx.Encode() - //nolint:lll // chained - olderTcc, isMatch := ss.handlerCtx.GetNamespaceCollection().GetAnalyticsCacheTableNamespaceConfigurator().Match(tableName, reqEncoding, ss.drmCfg.GetControlAttributes().GetControlLatestUpdateColumnName(), ss.drmCfg.GetControlAttributes().GetControlInsertEncodedIDColumnName()) - if isMatch { - nonControlColumns := ss.insertPreparedStatementCtx.GetNonControlColumns() - var nonControlColumnNames []string - for _, c := range nonControlColumns { - nonControlColumnNames = append(nonControlColumnNames, c.GetName()) - } - ss.handlerCtx.GetGarbageCollector().Update(tableName, olderTcc, currentTcc) - ss.insertionContainer.SetTableTxnCounters(tableName, olderTcc) - ss.insertPreparedStatementCtx.SetGCCtrlCtrs(olderTcc) - r, sqlErr := ss.handlerCtx.GetNamespaceCollection().GetAnalyticsCacheTableNamespaceConfigurator().Read( - tableName, reqEncoding, - ss.drmCfg.GetControlAttributes().GetControlInsertEncodedIDColumnName(), - nonControlColumnNames) - if sqlErr != nil { - internaldto.NewErroneousExecutorOutput(sqlErr) - } - ss.drmCfg.ExtractObjectFromSQLRows(r, nonControlColumns, ss.stream) - return internaldto.NewEmptyExecutorOutput() - } - transformType, transformBody := extractGraphQLResponseTransform(ss.tableMeta) - cursorCfg := buildGraphQLCursorConfig(gql, cursorJsonPath) - graphQLReader, err := graphql.NewStandardGQLReaderFull( - client, - req, - ss.handlerCtx.GetRuntimeContext().HTTPPageLimit, - gql.GetQuery(), - paramMap, - "", - responseJsonPath, - cursorCfg, - transformType, - transformBody, - ) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - for { - response, readErr := graphQLReader.Read() - ss.handlerCtx.LogHTTPResponseMap(response) - if len(response) > 0 { - if processErr := ss.processGraphQLPage(response, tableName, &housekeepingDone); processErr != nil { - return internaldto.NewErroneousExecutorOutput(processErr) - } - } - if errors.Is(readErr, io.EOF) { - break - } - if readErr != nil { - return internaldto.NewErroneousExecutorOutput(readErr) - } - } - } - return internaldto.NewEmptyExecutorOutput() - } - - prep := func() drm.PreparedStatementCtx { - return ss.insertPreparedStatementCtx - } - insertPrim := primitive.NewGenericPrimitive( - ex, - prep, - ss.txnCtrlCtr, - primitive_context.NewPrimitiveContext(), - ) - graph := ss.graph - insertNode := graph.CreatePrimitiveNode(insertPrim) - ss.root = insertNode - - return nil -} - -func (ss *GraphQLSingleSelectAcquire) processGraphQLPage( - response []map[string]interface{}, - tableName string, - housekeepingDone *bool, -) error { - if !*housekeepingDone && ss.insertPreparedStatementCtx != nil { - _, hkErr := ss.handlerCtx.GetSQLEngine().Exec(ss.insertPreparedStatementCtx.GetGCHousekeepingQueries()) - //nolint:errcheck // pre-existing behaviour: housekeeping counter set is best-effort - ss.insertionContainer.SetTableTxnCounters(tableName, ss.insertPreparedStatementCtx.GetGCCtrlCtrs()) - *housekeepingDone = true - if hkErr != nil { - return hkErr - } - } - if writeErr := ss.stream.Write(response); writeErr != nil { - return writeErr - } - for _, item := range response { - // TODO: handle request encoding - r, insertErr := ss.drmCfg.ExecuteInsertDML(ss.handlerCtx.GetSQLEngine(), ss.insertPreparedStatementCtx, item, "") - logging.GetLogger().Infoln(fmt.Sprintf("insert result = %v, error = %v", r, insertErr)) - if insertErr != nil { - return insertErr - } - } - return nil -} - -func extractGraphQLResponseTransform(tableMeta tablemetadata.ExtendedTableMetadata) (string, string) { - op, err := tableMeta.GetMethod() - if err != nil || op == nil { - return "", "" - } - er, ok := op.GetResponse() - if !ok || er == nil { - return "", "" - } - t, ok := er.GetTransform() - if !ok || t == nil { - return "", "" - } - return t.GetType(), t.GetBody() -} - -func buildGraphQLCursorConfig(gql formulation.GraphQL, cursorJSONPath string) graphql.CursorConfig { - cfg := graphql.CursorConfig{JSONPath: cursorJSONPath} - if strategy, ok := gql.GetCursorStrategy(); ok && strategy != "" { - cfg.Strategy = graphql.CursorStrategy(strategy) - } - if format, ok := gql.GetCursorFormat(); ok && format != "" { - cfg.FormatTemplate = format - } - if terminator, ok := gql.GetCursorTerminateOnJSONPath(); ok && terminator != "" { - cfg.TerminateOnJSONPath = terminator - } - if pageSize, ok := gql.GetCursorPageSize(); ok && pageSize > 0 { - cfg.PageSize = pageSize - } - return cfg -} diff --git a/internal/stackql/primitivebuilder/insert_or_update.go b/internal/stackql/primitivebuilder/insert_or_update.go index 93964816a..9ed0fd447 100644 --- a/internal/stackql/primitivebuilder/insert_or_update.go +++ b/internal/stackql/primitivebuilder/insert_or_update.go @@ -76,9 +76,7 @@ func (ss *insertOrUpdate) Build() error { return fmt.Errorf("mutation executor: cannot accomodate verb '%s'", mutableInput.GetVerb()) } } else { - genericBldr, genericBldrSetupErr = newGenericHTTPStreamInput( - mutableInput, - ) + return fmt.Errorf("provider mutations must be planned through omnisdk") } if genericBldrSetupErr != nil { diff --git a/internal/stackql/primitivebuilder/maps_aggregator_dto.go b/internal/stackql/primitivebuilder/maps_aggregator_dto.go deleted file mode 100644 index cf8a287ce..000000000 --- a/internal/stackql/primitivebuilder/maps_aggregator_dto.go +++ /dev/null @@ -1,33 +0,0 @@ -package primitivebuilder - -var ( - _ mapsAggregatorDTO = (*standardMapsAggregatorDTO)(nil) -) - -type mapsAggregatorDTO interface { - getParameterMap() map[int]map[string]interface{} - // getInputMap() map[int]map[int]interface{} -} - -type standardMapsAggregatorDTO struct { - parameterMap map[int]map[string]interface{} - inputMap map[int]map[int]interface{} -} - -func (dto *standardMapsAggregatorDTO) getParameterMap() map[int]map[string]interface{} { - return dto.parameterMap -} - -// func (dto *standardMapsAggregatorDTO) getInputMap() map[int]map[int]interface{} { -// return dto.inputMap -// } - -func newMapsAggregatorDTO( - parameterMap map[int]map[string]interface{}, - inputMap map[int]map[int]interface{}, -) mapsAggregatorDTO { - return &standardMapsAggregatorDTO{ - parameterMap: parameterMap, - inputMap: inputMap, - } -} diff --git a/internal/stackql/primitivebuilder/mono_valent_builder.go b/internal/stackql/primitivebuilder/mono_valent_builder.go deleted file mode 100644 index c69dfe109..000000000 --- a/internal/stackql/primitivebuilder/mono_valent_builder.go +++ /dev/null @@ -1,122 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/execution" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" -) - -// monoValentBuilder implements the Builder interface -// and represents the action of acquiring data from an endpoint -// and then persisting that data into a table. -// This data would then subsequently be queried by later execution phases. -type monoValentBuilder struct { - graphHolder primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - tableMeta tablemetadata.ExtendedTableMetadata - drmCfg drm.Config - insertPreparedStatementCtx drm.PreparedStatementCtx - insertionContainer tableinsertioncontainer.TableInsertionContainer - txnCtrlCtr internaldto.TxnControlCounters - rowSort func(map[string]map[string]interface{}) []string - root primitivegraph.PrimitiveNode - stream streaming.MapStream - isReadOnly bool //nolint:unused // TODO: build out - isAwait bool //nolint:unused // TODO: build out - monoValentExecutorFactory execution.MonoValentExecutorFactory - bldrInput builder_input.BuilderInput -} - -func newMonoValentBuilder( - graphHolder primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - tableMeta tablemetadata.ExtendedTableMetadata, - insertCtx drm.PreparedStatementCtx, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, - isSkipResponse bool, - isMutation bool, - isAwait bool, - bldrInput builder_input.BuilderInput, -) Builder { - var tcc internaldto.TxnControlCounters - if insertCtx != nil { - tcc = insertCtx.GetGCCtrlCtrs() - } - if stream == nil { - stream = streaming.NewNopMapStream() - } - return &monoValentBuilder{ - graphHolder: graphHolder, - handlerCtx: handlerCtx, - tableMeta: tableMeta, - rowSort: rowSort, - drmCfg: handlerCtx.GetDrmConfig(), - insertPreparedStatementCtx: insertCtx, - insertionContainer: insertionContainer, - txnCtrlCtr: tcc, - stream: stream, - bldrInput: bldrInput, - monoValentExecutorFactory: execution.NewMonoValentExecutorFactory( - graphHolder, - handlerCtx, - tableMeta, - insertCtx, - insertionContainer, - rowSort, - stream, - isSkipResponse, - isMutation, - isAwait, - bldrInput, - ), - } -} - -func (mv *monoValentBuilder) GetRoot() primitivegraph.PrimitiveNode { - return mv.root -} - -func (mv *monoValentBuilder) GetTail() primitivegraph.PrimitiveNode { - return mv.root -} - -func (mv *monoValentBuilder) Build() error { - tableName, err := mv.tableMeta.GetTableName() - if err != nil { - return err - } - ex, err := mv.monoValentExecutorFactory.GetExecutor() - - if err != nil { - return err - } - - prep := func() drm.PreparedStatementCtx { - return mv.insertPreparedStatementCtx - } - primitiveCtx := primitive_context.NewPrimitiveContext() - primitiveCtx.SetIsReadOnly(true) - insertPrim := primitive.NewGenericPrimitive( - ex, - prep, - mv.txnCtrlCtr, - primitiveCtx, - ).WithDebugName(fmt.Sprintf("insert_%s_%s", tableName, mv.tableMeta.GetAlias())) - graphHolder := mv.graphHolder - insertNode := graphHolder.CreatePrimitiveNode(insertPrim) - mv.root = insertNode - - return nil -} diff --git a/internal/stackql/primitivebuilder/native_select.go b/internal/stackql/primitivebuilder/native_select.go index 3c35c0f2d..7d2482759 100644 --- a/internal/stackql/primitivebuilder/native_select.go +++ b/internal/stackql/primitivebuilder/native_select.go @@ -6,10 +6,10 @@ import ( "io" "strconv" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/nativedb" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" diff --git a/internal/stackql/primitivebuilder/nop.go b/internal/stackql/primitivebuilder/nop.go index 4cfa76186..1e5f745ac 100644 --- a/internal/stackql/primitivebuilder/nop.go +++ b/internal/stackql/primitivebuilder/nop.go @@ -1,11 +1,11 @@ package primitivebuilder import ( - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/util" ) diff --git a/internal/stackql/primitivebuilder/raw_native_exec.go b/internal/stackql/primitivebuilder/raw_native_exec.go index 099d34a23..3207b6ffa 100644 --- a/internal/stackql/primitivebuilder/raw_native_exec.go +++ b/internal/stackql/primitivebuilder/raw_native_exec.go @@ -3,10 +3,10 @@ package primitivebuilder import ( "fmt" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/util" diff --git a/internal/stackql/primitivebuilder/raw_native_select.go b/internal/stackql/primitivebuilder/raw_native_select.go index ddb3e4426..36a3dde75 100644 --- a/internal/stackql/primitivebuilder/raw_native_select.go +++ b/internal/stackql/primitivebuilder/raw_native_select.go @@ -3,10 +3,10 @@ package primitivebuilder import ( "fmt" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/data_staging/input_data_staging" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" ) diff --git a/internal/stackql/primitivebuilder/shortcuts.go b/internal/stackql/primitivebuilder/shortcuts.go index ddeac4a96..a48f2f51e 100644 --- a/internal/stackql/primitivebuilder/shortcuts.go +++ b/internal/stackql/primitivebuilder/shortcuts.go @@ -6,9 +6,6 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/buildinfo" @@ -16,13 +13,10 @@ import ( "github.com/stackql/stackql/internal/stackql/dependencies" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/metadatavisitors" + "github.com/stackql/stackql/internal/stackql/intrinsic" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/provider" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/internal/stackql/util" - "github.com/stackql/stackql/pkg/prettyprint" ) func NewUpdateableValsPrimitive( @@ -99,170 +93,6 @@ func NewInsertableValsPrimitive( }), nil } -//nolint:funlen,gocognit // permissable -func NewShowInstructionExecutor( - node *sqlparser.Show, - prov provider.IProvider, - tbl tablemetadata.ExtendedTableMetadata, - handlerCtx handler.HandlerContext, - commentDirectives sqlparser.CommentDirectives, - tableFilter func(formulation.ITable, - ) (formulation.ITable, error), -) internaldto.ExecutorOutput { - extended := strings.TrimSpace(strings.ToUpper(node.Extended)) == "EXTENDED" - nodeTypeUpperCase := strings.ToUpper(node.Type) - var keys map[string]map[string]interface{} - var columnOrder []string - var err error - var filter func(interface{}) (formulation.ITable, error) - logging.GetLogger().Infoln(fmt.Sprintf("filter type = %T", filter)) - switch nodeTypeUpperCase { - case "AUTH": - logging.GetLogger().Infoln(fmt.Sprintf("Show For node.Type = '%s'", node.Type)) - authCtx, authErr := handlerCtx.GetAuthContext(prov.GetProviderString()) - if authErr == nil { - var authMeta *formulation.AuthMetadata - authMeta, err = prov.ShowAuth(authCtx) - if err == nil { - keys = map[string]map[string]interface{}{ - "1": authMeta.ToMap(), - } - columnOrder = authMeta.GetHeaders() - } - } - case "INSERT": - stmtKeys, errOut, ok := buildInsertShowOutput(node, tbl, handlerCtx, commentDirectives, extended) - if !ok { - return errOut - } - keys = stmtKeys - case "METHODS": - var rsc formulation.Resource - rsc, err = prov.GetResource( - node.OnTable.Qualifier.GetRawVal(), - node.OnTable.Name.GetRawVal(), - handlerCtx.GetRuntimeContext()) - if err != nil { - return util.PrepareResultSet(internaldto.NewPrepareResultSetDTO(nil, keys, columnOrder, nil, err, nil, - handlerCtx.GetTypingConfig())) - } - methods := rsc.GetMethodsMatched() - var filter func(formulation.ITable) (formulation.ITable, error) - if tbl == nil { - logging.GetLogger().Infoln( - "table and therefore filter not found for AST, shall procede nil filter") - } else { - filter = tbl.GetTableFilter() - } - methods, err = filterMethods(methods, filter) - if err != nil { - return util.PrepareResultSet(internaldto.NewPrepareResultSetDTO(nil, keys, columnOrder, nil, err, nil, - handlerCtx.GetTypingConfig())) - } - mOrd, mErr := methods.OrderMethods() - if mErr != nil { - return util.PrepareResultSet(internaldto.NewPrepareResultSetDTO(nil, keys, columnOrder, nil, mErr, nil, - handlerCtx.GetTypingConfig())) - } - views, hasViews := rsc.GetViewsForSqlDialect( - handlerCtx.GetSQLSystem().GetName(), - ) - methodKeys := make(map[string]map[string]interface{}) - j := 0 - if hasViews { - for i, v := range views { - vMap := v.ToPresentationMap(extended) - if i == 0 { - methodKeys[fmt.Sprintf("%06d", i)] = vMap - j++ - columnOrder = v.GetColumnOrder(extended) - } - } - } - for i, k := range mOrd { - method := k - methMap := method.ToPresentationMap(extended) - methodKeys[fmt.Sprintf("%06d", j+i)] = methMap - columnOrder = method.GetColumnOrder(extended) - } - keys = methodKeys - case "PROVIDERS": - keys, err = handlerCtx.GetSupportedProviders(extended) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - rv := util.PrepareResultSet(internaldto.NewPrepareResultSetDTO(nil, keys, columnOrder, nil, err, nil, - handlerCtx.GetTypingConfig())) - if len(keys) == 0 { - rv = util.EmptyProtectResultSet( - rv, - []string{"name", "version"}, - handlerCtx.GetTypingConfig(), - ) - } - return rv - case "RESOURCES": - svcName := node.OnTable.Name.GetRawVal() - if svcName == "" { - return prepareErroneousResultSet( - keys, - columnOrder, - fmt.Errorf("no service designated from which to resolve resources"), - handlerCtx.GetTypingConfig(), - ) - } - var resources map[string]formulation.Resource - resources, err = prov.GetResourcesRedacted(svcName, handlerCtx.GetRuntimeContext(), extended) - if err != nil { - return prepareErroneousResultSet(keys, columnOrder, err, - handlerCtx.GetTypingConfig()) - } - columnOrder = formulation.GetResourcesHeader(extended) - var filter func(formulation.ITable) (formulation.ITable, error) - filter = tableFilter - resources, err = filterResources(resources, filter) - if err != nil { - return prepareErroneousResultSet(keys, columnOrder, err, - handlerCtx.GetTypingConfig(), - ) - } - keys = make(map[string]map[string]interface{}) - for k, v := range resources { - keys[k] = v.ToMap(extended) - } - case "SERVICES": - logging.GetLogger().Infoln( - fmt.Sprintf( - "Show For node.Type = '%s': Displaying services for provider = '%s'", - node.Type, - prov.GetProviderString(), - ), - ) - var services map[string]formulation.ProviderService - services, err = prov.GetProviderServicesRedacted(handlerCtx.GetRuntimeContext(), extended) - if err != nil { - return prepareErroneousResultSet(keys, columnOrder, err, - handlerCtx.GetTypingConfig(), - ) - } - columnOrder = formulation.GetServicesHeader(extended) - services, err = filterServices(services, tableFilter, handlerCtx.GetRuntimeContext().UseNonPreferredAPIs) - if err != nil { - return prepareErroneousResultSet(keys, columnOrder, err, - handlerCtx.GetTypingConfig()) - } - keys = convertProviderServicesToMap(services, extended) - case "VERSION": - columnOrder, keys = buildVersionShowOutput(extended) - case "CONTRIBUTORS": - columnOrder, keys = buildContributorsShowOutput(extended) - case "DEPENDENCIES": - return buildDependenciesShowOutput(node, extended, handlerCtx) - } - return util.PrepareResultSet(internaldto.NewPrepareResultSetDTO(nil, keys, columnOrder, nil, err, nil, - handlerCtx.GetTypingConfig())) -} - func buildDependenciesShowOutput( node *sqlparser.Show, extended bool, handlerCtx handler.HandlerContext, ) internaldto.ExecutorOutput { @@ -314,69 +144,6 @@ func (row *dependencySQLRow) GetRowDataForPgWire() []interface{} { return values } -// buildInsertShowOutput renders the SHOW INSERT result for a resource method. -// On success it returns the rows map, a zero-value ExecutorOutput, and true. -// On any resolver / formatter failure it returns nil, an erroneous output, -// and false; callers should propagate that output to the caller directly. -// Extracted from NewShowInstructionExecutor to keep gocyclo under threshold. -func buildInsertShowOutput( - node *sqlparser.Show, - tbl tablemetadata.ExtendedTableMetadata, - handlerCtx handler.HandlerContext, - commentDirectives sqlparser.CommentDirectives, - extended bool, -) (map[string]map[string]interface{}, internaldto.ExecutorOutput, bool) { - ppCtx := prettyprint.NewPrettyPrintContext( - handlerCtx.GetRuntimeContext().OutputFormat == constants.PrettyTextStr, - constants.DefaultPrettyPrintIndent, - constants.DefaultPrettyPrintBaseIndent, - "'", - logging.GetLogger(), - ) - meth, methErr := tbl.GetMethod() - if methErr != nil { - tblName, tblErr := tbl.GetStackQLTableName() - if tblErr != nil { - return nil, util.GenerateSimpleErroneousOutput( - fmt.Errorf( - "cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS: %w", //nolint:lll // prescribed message - methErr), - handlerCtx.GetTypingConfig(), - ), false - } - return nil, util.GenerateSimpleErroneousOutput( - fmt.Errorf( - "cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS IN %s: %w", //nolint:lll // prescribed message - tblName, methErr), - handlerCtx.GetTypingConfig(), - ), false - } - svc, svcErr := tbl.GetService() - if svcErr != nil { - return nil, util.GenerateSimpleErroneousOutput(svcErr, - handlerCtx.GetTypingConfig(), - ), false - } - pp := prettyprint.NewPrettyPrinter(ppCtx) - ppPlaceholder := prettyprint.NewPrettyPrinter(ppCtx) - requiredOnly := commentDirectives != nil && commentDirectives.IsSet("REQUIRED") - insertStmt, insertErr := metadatavisitors.ToInsertStatement( - node.Columns, meth, svc, extended, pp, ppPlaceholder, requiredOnly) - tableName, _ := tbl.GetTableName() - if insertErr != nil { - return nil, util.GenerateSimpleErroneousOutput( - fmt.Errorf("error creating insert statement for %s: %w", tableName, insertErr), - handlerCtx.GetTypingConfig(), - ), false - } - stmtStr := fmt.Sprintf(insertStmt, tableName) - return map[string]map[string]interface{}{ - "1": { - "insert_statement": stmtStr, - }, - }, nil, true -} - // buildVersionShowOutput renders the SHOW VERSION result. Extracted so that // NewShowInstructionExecutor stays under the gocyclo threshold. func buildVersionShowOutput(extended bool) ([]string, map[string]map[string]interface{}) { @@ -415,233 +182,6 @@ func buildContributorsShowOutput(extended bool) ([]string, map[string]map[string return []string{"contributor"}, keys } -//nolint:errcheck // future proofing -func filterResources( - resources map[string]formulation.Resource, - tableFilter func(formulation.ITable) (formulation.ITable, error), -) (map[string]formulation.Resource, error) { - var err error - if tableFilter != nil { - filteredResources := make(map[string]formulation.Resource) - for k, rsc := range resources { - filteredResource, filterErr := tableFilter(rsc) - if filterErr == nil && filteredResource != nil { - filteredResources[k] = filteredResource.(formulation.Resource) - } - if filterErr != nil { - err = filterErr - } - } - resources = filteredResources - } - return resources, err -} - -func getProviderServiceMap(item formulation.ProviderService, extended bool) map[string]interface{} { - retVal := map[string]interface{}{ - "id": item.GetID(), - "name": item.GetName(), - "title": item.GetTitle(), - } - if extended { - retVal["description"] = item.GetDescription() - retVal["version"] = item.GetVersion() - } - return retVal -} - -func convertProviderServicesToMap( - services map[string]formulation.ProviderService, - extended bool, -) map[string]map[string]interface{} { - retVal := make(map[string]map[string]interface{}) - for k, v := range services { - retVal[k] = getProviderServiceMap(v, extended) - } - return retVal -} - -//nolint:errcheck // future proofing -func filterServices( - services map[string]formulation.ProviderService, - tableFilter func(formulation.ITable) (formulation.ITable, error), - useNonPreferredAPIs bool, -) (map[string]formulation.ProviderService, error) { - var err error - //nolint:nestif // TODO: refactor - if tableFilter != nil { - filteredServices := make(map[string]formulation.ProviderService) - for k, svc := range services { - if useNonPreferredAPIs || svc.IsPreferred() { - filteredService, filterErr := tableFilter(svc) - if filterErr == nil && filteredService != nil { - filteredServices[k] = (filteredService.(formulation.ProviderService)) - } - if filterErr != nil { - err = filterErr - } - } - } - services = filteredServices - } - return services, err -} - -func filterMethods( - methods formulation.Methods, - tableFilter func(formulation.ITable) (formulation.ITable, error), -) (formulation.Methods, error) { - var err error - if tableFilter != nil { - filteredMethods := formulation.EmptyMethods() - for _, m := range methods.List() { - k := m.GetKey() - pm := m.GetValue() - filteredMethod, filterErr := tableFilter(pm) - if filterErr == nil && filteredMethod != nil { - //nolint:errcheck // letting it go for now, but should be addressed - filteredMethods.Put(k, pm) - } - if filterErr != nil { - err = filterErr - } - } - methods = filteredMethods - } - return methods, err -} - -func prepareErroneousResultSet( - rowMap map[string]map[string]interface{}, //nolint:unparam // future proofing - columnOrder []string, - err error, - typCfg typing.Config, -) internaldto.ExecutorOutput { - return util.PrepareResultSet( - internaldto.NewPrepareResultSetDTO( - nil, - rowMap, - columnOrder, - nil, - err, - nil, - typCfg, - ), - ) -} - -func NewDescribeTableInstructionExecutor( - handlerCtx handler.HandlerContext, - tbl tablemetadata.ExtendedTableMetadata, - extended bool, - full bool, //nolint:revive // future proofing -) internaldto.ExecutorOutput { - schema, err := tbl.GetSelectableObjectSchema() - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - columnOrder := formulation.GetDescribeHeader(extended) - descriptionMap := schema.ToDescriptionMap(extended) - keys := make(map[string]map[string]interface{}) - for k, v := range descriptionMap { - switch val := v.(type) { //nolint:gocritic // review later - case map[string]interface{}: - keys[k] = val - } - } - return util.PrepareResultSet( - internaldto.NewPrepareResultSetDTO( - nil, - keys, - columnOrder, - util.DescribeRowSort, - err, - nil, - handlerCtx.GetTypingConfig(), - ), - ) -} - -func NewDescribeMethodInstructionExecutor( - handlerCtx handler.HandlerContext, - rsc formulation.Resource, - methodName string, - extended bool, -) internaldto.ExecutorOutput { - mi, err := formulation.IntrospectMethod(rsc, methodName, extended) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - columnOrder := []string{"name", "type", "param_type", "shape"} - if extended { - columnOrder = append(columnOrder, "description") - } - rows := make(map[string]map[string]interface{}) - for i, f := range mi.GetFields() { - row := map[string]interface{}{ - "name": f.GetName(), - "type": f.GetType(), - "param_type": string(f.GetParamType()), - "shape": f.GetShape(), - } - if extended { - row["description"] = f.GetDescription() - } - rows[fmt.Sprintf("%06d", i)] = row - } - return util.PrepareResultSet( - internaldto.NewPrepareResultSetDTO( - nil, rows, columnOrder, nil, nil, nil, - handlerCtx.GetTypingConfig(), - ), - ) -} - -//nolint:revive // future proofing -func NewDescribeViewInstructionExecutor( - handlerCtx handler.HandlerContext, - tbl tablemetadata.ExtendedTableMetadata, - nonControlColumns []typing.ColumnMetadata, - extended bool, - full bool, -) internaldto.ExecutorOutput { - columnOrder := formulation.GetDescribeHeader(extended) - descriptionMap := columnsToFlatDescriptionMap(nonControlColumns, extended) - keys := make(map[string]map[string]interface{}) - for k, v := range descriptionMap { - switch val := v.(type) { //nolint:gocritic // TODO: review - case map[string]interface{}: - keys[k] = val - } - } - return util.PrepareResultSet( - internaldto.NewPrepareResultSetDTO( - nil, - keys, - columnOrder, - util.DescribeRowSort, - nil, - nil, - handlerCtx.GetTypingConfig(), - ), - ) -} - -func columnsToFlatDescriptionMap(colz []typing.ColumnMetadata, extended bool) map[string]interface{} { - retVal := make(map[string]interface{}) - for _, col := range colz { - colName := col.GetIdentifier() - colMap := make(map[string]interface{}) - colMap["name"] = colName - colMap["type"] = col.GetType() - if extended { - colMap["description"] = "" - } - retVal[colName] = colMap - } - return retVal -} - //nolint:revive // future proofing func NewLocalSelectExecutor( handlerCtx handler.HandlerContext, @@ -677,3 +217,49 @@ func NewLocalSelectExecutor( )) }), nil } + +func NewShowInstructionExecutor( + node *sqlparser.Show, prov provider.IProvider, handlerCtx handler.HandlerContext, +) internaldto.ExecutorOutput { + if execute, handled := intrinsic.GeneratePrimitiveFunc(handlerCtx, node); handled { + return execute() + } + extended := strings.EqualFold(strings.TrimSpace(node.Extended), "extended") + var rows map[string]map[string]interface{} + var columns []string + switch strings.ToUpper(node.Type) { + case "AUTH": + authCtx, err := handlerCtx.GetAuthContext(prov.GetProviderString()) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + metadata, err := prov.ShowAuth(authCtx) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows, columns = map[string]map[string]interface{}{"1": metadata.ToMap()}, []string{"principal", "type", "source"} + case "PROVIDERS": + var err error + rows, err = handlerCtx.GetSupportedProviders(extended) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + if len(rows) == 0 { + columns = []string{"name", "version"} + } + case "VERSION": + columns, rows = buildVersionShowOutput(extended) + case "CONTRIBUTORS": + columns, rows = buildContributorsShowOutput(extended) + case "DEPENDENCIES": + return buildDependenciesShowOutput(node, extended, handlerCtx) + default: + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("unsupported SHOW %s", node.Type)) + } + result := util.PrepareResultSet(internaldto.NewPrepareResultSetDTO( + nil, rows, columns, nil, nil, nil, handlerCtx.GetTypingConfig())) + if len(rows) == 0 { + return util.EmptyProtectResultSet(result, columns, handlerCtx.GetTypingConfig()) + } + return result +} diff --git a/internal/stackql/primitivebuilder/single_acquire_and_select.go b/internal/stackql/primitivebuilder/single_acquire_and_select.go deleted file mode 100644 index 123a35b8e..000000000 --- a/internal/stackql/primitivebuilder/single_acquire_and_select.go +++ /dev/null @@ -1,82 +0,0 @@ -package primitivebuilder - -import ( - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" -) - -type SingleAcquireAndSelect struct { - graph primitivegraph.PrimitiveGraphHolder - acquireBuilder Builder - selectBuilder Builder - bldrInput builder_input.BuilderInput - root primitivegraph.PrimitiveNode -} - -func NewSingleAcquireAndSelect( - // graph primitivegraph.PrimitiveGraphHolder, - // txnControlCounters internaldto.TxnControlCounters, //nolint:revive // future proofing - // handlerCtx handler.HandlerContext, - // insertContainer tableinsertioncontainer.TableInsertionContainer, - bldrInput builder_input.BuilderInput, - insertCtx drm.PreparedStatementCtx, - selectCtx drm.PreparedStatementCtx, - rowSort func(map[string]map[string]interface{}) []string, -) Builder { - graph, _ := bldrInput.GetGraphHolder() - // txnControlCounters, _ := bldrInput.GetTxnCtrlCtrs() - handlerCtx, _ := bldrInput.GetHandlerContext() - insertContainer, _ := bldrInput.GetTableInsertionContainer() - return &SingleAcquireAndSelect{ - graph: graph, - acquireBuilder: NewSingleSelectAcquire( - graph, - handlerCtx, - insertContainer, - insertCtx, - rowSort, - nil, - bldrInput, - ), - selectBuilder: NewSingleSelect( - graph, handlerCtx, selectCtx, - []tableinsertioncontainer.TableInsertionContainer{insertContainer}, - rowSort, - streaming.NewNopMapStream()), - bldrInput: bldrInput, - } -} - -func (ss *SingleAcquireAndSelect) GetRoot() primitivegraph.PrimitiveNode { - return ss.acquireBuilder.GetRoot() -} - -func (ss *SingleAcquireAndSelect) GetTail() primitivegraph.PrimitiveNode { - return ss.selectBuilder.GetTail() -} - -func (ss *SingleAcquireAndSelect) Build() error { - err := ss.acquireBuilder.Build() - if err != nil { - return err - } - err = ss.selectBuilder.Build() - if err != nil { - return err - } - graph := ss.graph - graph.NewDependency(ss.acquireBuilder.GetTail(), ss.selectBuilder.GetRoot(), 1.0) - rootNode := ss.acquireBuilder.GetRoot() - ss.root = rootNode - dependencyNode, dependencyNodeExists := ss.bldrInput.GetDependencyNode() - if dependencyNodeExists { - //nolint:errcheck // TODO: fix this - rootNode.SetInputAlias("", dependencyNode.ID()) - ss.graph.NewDependency(dependencyNode, rootNode, 1.0) - // ss.root = dependencyNode // dont think this is needed - } - return nil -} diff --git a/internal/stackql/primitivebuilder/single_select.go b/internal/stackql/primitivebuilder/single_select.go deleted file mode 100644 index e16df737d..000000000 --- a/internal/stackql/primitivebuilder/single_select.go +++ /dev/null @@ -1,92 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/data_staging/output_data_staging" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" -) - -type SingleSelect struct { - graph primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - drmCfg drm.Config - selectPreparedStatementCtx drm.PreparedStatementCtx - insertionContainers []tableinsertioncontainer.TableInsertionContainer - txnCtrlCtr internaldto.TxnControlCounters - rowSort func(map[string]map[string]interface{}) []string - root primitivegraph.PrimitiveNode - stream streaming.MapStream -} - -func NewSingleSelect( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - selectCtx drm.PreparedStatementCtx, - insertionContainers []tableinsertioncontainer.TableInsertionContainer, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, -) Builder { - return &SingleSelect{ - graph: graph, - handlerCtx: handlerCtx, - rowSort: rowSort, - drmCfg: handlerCtx.GetDrmConfig(), - selectPreparedStatementCtx: selectCtx, - insertionContainers: insertionContainers, - txnCtrlCtr: selectCtx.GetGCCtrlCtrs(), - stream: stream, - } -} - -func (ss *SingleSelect) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *SingleSelect) GetTail() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *SingleSelect) Build() error { - //nolint:revive // acceptable for now - selectEx := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - // select phase - logging.GetLogger().Infoln( - fmt.Sprintf( - "running select with control parameters: %v", - ss.selectPreparedStatementCtx.GetGCCtrlCtrs(), - ), - ) - - outputter := output_data_staging.NewNaiveOutputter( - output_data_staging.NewNaivePacketPreparator( - output_data_staging.NewNaiveSource( - ss.handlerCtx.GetSQLEngine(), - drm.NewPreparedStatementParameterized(ss.selectPreparedStatementCtx, nil, true), - ss.drmCfg, - ), - ss.selectPreparedStatementCtx.GetNonControlColumns(), - ss.stream, - ss.drmCfg, - ), - ss.selectPreparedStatementCtx.GetNonControlColumns(), - ss.handlerCtx.GetTypingConfig(), - ) - // TODO: consider deep copy for output and hnadle errors - rv := outputter.OutputExecutorResult() - _ = ss.graph.SetExecutorOutput("select", rv) - return rv - } - graph := ss.graph - selectNode := graph.CreatePrimitiveNode(primitive.NewLocalPrimitive(selectEx)) - ss.root = selectNode - - return nil -} diff --git a/internal/stackql/primitivebuilder/single_select_acquire.go b/internal/stackql/primitivebuilder/single_select_acquire.go deleted file mode 100644 index 5d714bcf1..000000000 --- a/internal/stackql/primitivebuilder/single_select_acquire.go +++ /dev/null @@ -1,49 +0,0 @@ -package primitivebuilder - -import ( - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" -) - -func NewSingleSelectAcquire( - graphHolder primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - insertCtx drm.PreparedStatementCtx, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, - bldrInput builder_input.BuilderInput, -) Builder { - isAwait := bldrInput.IsAwait() - tableMeta := insertionContainer.GetTableMetadata() - _, isGraphQL := tableMeta.GetGraphQL() - if isGraphQL { - return newGraphQLSingleSelectAcquire( - graphHolder, - handlerCtx, - tableMeta, - insertCtx, - insertionContainer, - rowSort, - stream, - bldrInput, - ) - } - return newMonoValentBuilder( - graphHolder, - handlerCtx, - tableMeta, - insertCtx, - insertionContainer, - rowSort, - stream, - false, - false, - isAwait, - bldrInput, - ) -} diff --git a/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go b/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go deleted file mode 100644 index e504bdc6b..000000000 --- a/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go +++ /dev/null @@ -1,148 +0,0 @@ -package primitivebuilder - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/stackql/internal/stackql/data_staging/input_data_staging" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" - "github.com/stackql/stackql/internal/stackql/primitive" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" -) - -// sqlDataSourceSingleSelectAcquire implements the Builder interface -// and represents the action of acquiring data from an endpoint -// and then persisting that data into a table. -// This data would then subsequently be queried by later execution phases. -type sqlDataSourceSingleSelectAcquire struct { - query string - queryArgs []interface{} - graph primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - tableMeta tablemetadata.ExtendedTableMetadata - drmCfg drm.Config - insertPreparedStatementCtx drm.PreparedStatementCtx - insertionContainer tableinsertioncontainer.TableInsertionContainer - txnCtrlCtr internaldto.TxnControlCounters - rowSort func(map[string]map[string]interface{}) []string - root primitivegraph.PrimitiveNode - stream streaming.MapStream -} - -func NewSQLDataSourceSingleSelectAcquire( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - query string, - queryArgs []interface{}, - insertCtx drm.PreparedStatementCtx, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, -) Builder { - tableMeta := insertionContainer.GetTableMetadata() - return newSQLDataSourceSingleSelectAcquire( - graph, - handlerCtx, - tableMeta, - insertCtx, - insertionContainer, - query, - queryArgs, - rowSort, - stream, - ) -} - -func newSQLDataSourceSingleSelectAcquire( - graph primitivegraph.PrimitiveGraphHolder, - handlerCtx handler.HandlerContext, - tableMeta tablemetadata.ExtendedTableMetadata, - insertCtx drm.PreparedStatementCtx, - insertionContainer tableinsertioncontainer.TableInsertionContainer, - query string, - queryArgs []interface{}, - rowSort func(map[string]map[string]interface{}) []string, - stream streaming.MapStream, -) Builder { - var tcc internaldto.TxnControlCounters - if insertCtx != nil { - tcc = insertCtx.GetGCCtrlCtrs() - } - if stream == nil { - stream = streaming.NewNopMapStream() - } - return &sqlDataSourceSingleSelectAcquire{ - graph: graph, - handlerCtx: handlerCtx, - tableMeta: tableMeta, - rowSort: rowSort, - drmCfg: handlerCtx.GetDrmConfig(), - insertPreparedStatementCtx: insertCtx, - insertionContainer: insertionContainer, - txnCtrlCtr: tcc, - stream: stream, - query: query, - queryArgs: queryArgs, - } -} - -func (ss *sqlDataSourceSingleSelectAcquire) GetRoot() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *sqlDataSourceSingleSelectAcquire) GetTail() primitivegraph.PrimitiveNode { - return ss.root -} - -func (ss *sqlDataSourceSingleSelectAcquire) Build() error { - sqlDB, ok := ss.tableMeta.GetSQLDataSource() - if !ok { - return fmt.Errorf("sql data source unavailable for sql data source query") - } - tableName, err := ss.tableMeta.GetTableName() - if err != nil { - return err - } - //nolint:revive // no big deal - ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - // ss.tableMeta.GetP - rows, rowsErr := sqlDB.Query(ss.query, ss.queryArgs...) - if rowsErr != nil { - return internaldto.NewErroneousExecutorOutput(rowsErr) - } - currentTcc := ss.insertPreparedStatementCtx.GetGCCtrlCtrs().Clone() - ss.graph.AddTxnControlCounters(currentTcc) - currentTcc.SetTableName(tableName) - //nolint:errcheck // TODO: fix - ss.insertionContainer.SetTableTxnCounters(tableName, currentTcc) - preparator := input_data_staging.NewNaiveNativeResultSetPreparator( - rows, - ss.handlerCtx.GetDrmConfig(), - ss.handlerCtx.GetTypingConfig(), - ss.insertPreparedStatementCtx, - ) - return preparator.PrepareNativeResultSet() - } - - prep := func() drm.PreparedStatementCtx { - return ss.insertPreparedStatementCtx - } - primitiveCtx := primitive_context.NewPrimitiveContext() - primitiveCtx.SetIsReadOnly(true) - insertPrim := primitive.NewGenericPrimitive( - ex, - prep, - ss.txnCtrlCtr, - primitiveCtx, - ) - graph := ss.graph - insertNode := graph.CreatePrimitiveNode(insertPrim) - ss.root = insertNode - - return nil -} diff --git a/internal/stackql/primitivebuilder/union.go b/internal/stackql/primitivebuilder/union.go index 6504e0afe..df6354247 100644 --- a/internal/stackql/primitivebuilder/union.go +++ b/internal/stackql/primitivebuilder/union.go @@ -1,13 +1,13 @@ package primitivebuilder import ( - "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql/internal/stackql/data_staging/output_data_staging" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivegraph" + "github.com/stackql/stackql/internal/stackql/streaming" ) type Union struct { diff --git a/internal/stackql/primitivecomposer/primitive_composer.go b/internal/stackql/primitivecomposer/primitive_composer.go index 3ac0ec5c1..1be69e6fb 100644 --- a/internal/stackql/primitivecomposer/primitive_composer.go +++ b/internal/stackql/primitivecomposer/primitive_composer.go @@ -1,10 +1,6 @@ package primitivecomposer import ( - "fmt" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -13,7 +9,7 @@ import ( "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/provider" "github.com/stackql/stackql/internal/stackql/sql_system" - "github.com/stackql/stackql/internal/stackql/suffix" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/symtab" "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/internal/stackql/taxonomy" @@ -27,9 +23,9 @@ import ( type PrimitiveComposer interface { AddChild(val PrimitiveComposer) AddIndirect(val PrimitiveComposer) - AssignParameters() (internaldto.TableParameterCollection, error) + ContainsSQLDataSource() bool - GetAssignedParameters() (internaldto.TableParameterCollection, bool) + GetAst() sqlparser.SQLNode GetASTFormatter() sqlparser.NodeFormatter GetBuilder() primitivebuilder.Builder @@ -53,7 +49,7 @@ type PrimitiveComposer interface { GetSymbol(k interface{}) (symtab.Entry, error) GetSymTab() symtab.SymTab GetTable(node sqlparser.SQLNode) (tablemetadata.ExtendedTableMetadata, error) - GetTableFilter() func(formulation.ITable) (formulation.ITable, error) + GetTables() taxonomy.TblMap GetTxnCounterManager() txncounter.Manager GetTxnCtrlCtrs() internaldto.TxnControlCounters @@ -84,7 +80,7 @@ type PrimitiveComposer interface { SetSymTab(symtab.SymTab) MergeSymTab(symtab.SymTab, string) error SetTable(node sqlparser.SQLNode, table tablemetadata.ExtendedTableMetadata) - SetTableFilter(tableFilter func(formulation.ITable) (formulation.ITable, error)) + SetTxnCtrlCtrs(tc internaldto.TxnControlCounters) SetUnionSelectPreparedStatementCtx(ctx drm.PreparedStatementCtx) SetValOnlyCols(m map[int]map[string]interface{}) @@ -115,7 +111,6 @@ type standardPrimitiveComposer struct { // needed globally for non-heirarchy queries, such as "SHOW SERVICES FROM google;" prov provider.IProvider - tableFilter func(formulation.ITable) (formulation.ITable, error) colsVisited map[string]bool likeAbleColumns []string @@ -152,8 +147,6 @@ type standardPrimitiveComposer struct { tccSetAheadOfTime bool - paramCollection internaldto.TableParameterCollection - isIndirect bool containsUserRelation bool @@ -203,10 +196,6 @@ func (pb *standardPrimitiveComposer) SetIsIndirect(isIndirect bool) { pb.isIndirect = isIndirect } -func (pb *standardPrimitiveComposer) GetAssignedParameters() (internaldto.TableParameterCollection, bool) { - return pb.paramCollection, pb.paramCollection != nil -} - func (pb *standardPrimitiveComposer) SetSymTab(symTab symtab.SymTab) { pb.symTab = symTab } @@ -258,97 +247,6 @@ func (pb *standardPrimitiveComposer) GetChildren() []PrimitiveComposer { return pb.children } -// withNativeCasingAliases adds each native-casing (snake_case) parameter alias to the same -// required/optional map as its wire parameter, so snake_case SQL keys resolve to the wire -// parameter. The alias and its wire key share an Addressable whose GetName is the wire name. -// any-sdk owns the casing translation; for methods without request native casing the -// supplied allWithCasing map equals the wire param set and this is a no-op. -func withNativeCasingAliases( - reqParams, optParams, allWithCasing map[string]formulation.Addressable, -) (map[string]formulation.Addressable, map[string]formulation.Addressable) { - for aliasKey, addr := range allWithCasing { - if _, isWire := reqParams[aliasKey]; isWire { - continue - } - if _, isWire := optParams[aliasKey]; isWire { - continue - } - if _, req := reqParams[addr.GetName()]; req { - reqParams[aliasKey] = addr - } else if _, opt := optParams[addr.GetName()]; opt { - optParams[aliasKey] = addr - } - } - return reqParams, optParams -} - -//nolint:gocognit // acceptable for now -func (pb *standardPrimitiveComposer) AssignParameters() (internaldto.TableParameterCollection, error) { - requiredParameters := suffix.NewParameterSuffixMap() - remainingRequiredParameters := suffix.NewParameterSuffixMap() - optionalParameters := suffix.NewParameterSuffixMap() - tbVisited := map[tablemetadata.ExtendedTableMetadata]struct{}{} - for _, tb := range pb.GetTables() { - if _, ok := tbVisited[tb]; ok { - continue - } - tbVisited[tb] = struct{}{} - tbID := tb.GetUniqueID() - var reqParams, tblOptParams, remainingReqParams map[string]formulation.Addressable - if view, isView := tb.GetIndirect(); isView { - // TODO: fill this out - assignedParams, ok := view.GetAssignedParameters() - if !ok { - continue - } - reqParams = assignedParams.GetRequiredParams().GetAll() - tblOptParams = assignedParams.GetOptionalParams().GetAll() - remainingReqParams = assignedParams.GetRemainingRequiredParams().GetAll() - } else { - // These methods need to incorporate request body parameters. - // Register native-casing (snake_case) aliases so snake_case WHERE / INSERT keys - // resolve to their wire parameter (any-sdk owns the casing translation). - reqParams, tblOptParams = withNativeCasingAliases( - tb.GetRequiredParameters(), - tb.GetOptionalParameters(), - tb.GetParametersIncludingNativeCasing(), - ) - } - // This method needs to incorporate request body parameters - - for k, v := range reqParams { - key := fmt.Sprintf("%s.%s", tbID, k) - _, keyExists := requiredParameters.Get(key) - if keyExists { - return nil, fmt.Errorf("key already is required: %s", k) - } - requiredParameters.Put(key, v) - } - for k, vOpt := range tblOptParams { - key := fmt.Sprintf("%s.%s", tbID, k) - _, keyExists := optionalParameters.Get(key) - if keyExists { - return nil, fmt.Errorf("key already is optional: %s", k) - } - optionalParameters.Put(key, vOpt) - } - for k, vRem := range remainingReqParams { - key := fmt.Sprintf("%s.%s", tbID, k) - if vRem == nil { - return nil, fmt.Errorf("remaining required parameter is nil: %s", k) - } - _, keyExists := remainingRequiredParameters.Get(key) - if !keyExists { - return nil, fmt.Errorf("key already is optional: %s", k) - } - remainingRequiredParameters.Delete(key) - } - } - rv := internaldto.NewTableParameterCollection(requiredParameters, optionalParameters, remainingRequiredParameters) - pb.paramCollection = rv - return rv, nil -} - func (pb *standardPrimitiveComposer) AddChild(val PrimitiveComposer) { pb.children = append(pb.children, val) } @@ -469,15 +367,6 @@ func (pb *standardPrimitiveComposer) SetColVisited(colname string, isVisited boo pb.colsVisited[colname] = isVisited } -func (pb *standardPrimitiveComposer) GetTableFilter() func(formulation.ITable) (formulation.ITable, error) { - return pb.tableFilter -} - -func (pb *standardPrimitiveComposer) SetTableFilter( - tableFilter func(formulation.ITable) (formulation.ITable, error)) { - pb.tableFilter = tableFilter -} - func (pb *standardPrimitiveComposer) SetInsertPreparedStatementCtx(ctx drm.PreparedStatementCtx) { pb.insertPreparedStatementCtx = ctx } diff --git a/internal/stackql/primitivegenerator/primitive_generator.go b/internal/stackql/primitivegenerator/primitive_generator.go index e456e0352..ea0a9c905 100644 --- a/internal/stackql/primitivegenerator/primitive_generator.go +++ b/internal/stackql/primitivegenerator/primitive_generator.go @@ -2,27 +2,18 @@ package primitivegenerator import ( "fmt" - "regexp" "strings" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/iqlutil" - "github.com/stackql/stackql/internal/stackql/parserutil" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/planbuilderinput" "github.com/stackql/stackql/internal/stackql/primitivebuilder" "github.com/stackql/stackql/internal/stackql/primitivecomposer" "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/relational" "github.com/stackql/stackql/internal/stackql/symtab" - "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/pkg/sqltypeutil" - - "github.com/stackql/stackql-parser/go/sqltypes" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -40,12 +31,6 @@ type PrimitiveGenerator interface { AnalyzeRegistry(pbi planbuilderinput.PlanBuilderInput) error AnalyzeSelectStatement(pbi planbuilderinput.PlanBuilderInput) error AnalyzeStatement(pbi planbuilderinput.PlanBuilderInput) error - AnalyzeUnaryExec( - pbi planbuilderinput.PlanBuilderInput, - handlerCtx handler.HandlerContext, - node *sqlparser.Exec, - selectNode *sqlparser.Select, - cols []parserutil.ColumnHandle) (tablemetadata.ExtendedTableMetadata, error) CreateIndirectPrimitiveGenerator(ast sqlparser.SQLNode, handlerCtx handler.HandlerContext) PrimitiveGenerator GetPrimitiveComposer() primitivecomposer.PrimitiveComposer SetIsIndirect(isIndirect bool) @@ -179,73 +164,6 @@ func (pb *standardPrimitiveGenerator) AddChildPrimitiveGenerator( return retVal } -//nolint:unparam,revive // acceptable -func (pb *standardPrimitiveGenerator) comparisonExprToFilterFunc( - table formulation.ITable, - parentNode *sqlparser.Show, - expr *sqlparser.ComparisonExpr, -) (func(formulation.ITable) (formulation.ITable, error), error) { - qualifiedName, ok := expr.Left.(*sqlparser.ColName) - if !ok { - return nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) - } - if !qualifiedName.Qualifier.IsEmpty() { - return nil, fmt.Errorf("unsupported qualifier for column: %v", sqlparser.String(qualifiedName)) - } - colName := qualifiedName.Name.GetRawVal() - tableContainsKey := table.KeyExists(colName) - if !tableContainsKey { - return nil, fmt.Errorf("col name = '%s' not found in table name = '%s'", colName, table.GetName()) - } - _, lhsValErr := table.GetKeyAsSqlVal(colName) - if lhsValErr != nil { - return nil, lhsValErr - } - var resolved sqltypes.Value - var rhsStr string - switch right := expr.Right.(type) { - case *sqlparser.SQLVal: - if right.Type != sqlparser.IntVal && right.Type != sqlparser.StrVal { - return nil, fmt.Errorf("unexpected: %v", sqlparser.String(expr)) - } - pv, err := sqlparser.NewPlanValue(right) - if err != nil { - return nil, err - } - rhsStr = string(right.Val) - resolved, err = pv.ResolveValue(nil) - if err != nil { - return nil, err - } - case sqlparser.BoolVal: - var resErr error - resolved, resErr = sqltypeutil.InterfaceToSQLType(right == true) //nolint:gosimple // test later - if resErr != nil { - return nil, resErr - } - default: - return nil, fmt.Errorf("unexpected: %v", sqlparser.String(right)) - } - var retVal func(formulation.ITable) (formulation.ITable, error) - if expr.Operator == sqlparser.LikeStr || expr.Operator == sqlparser.NotLikeStr { - likeRegexp, err := regexp.Compile(iqlutil.TranslateLikeToRegexPattern(rhsStr)) - if err != nil { - return nil, err - } - retVal = relational.ConstructLikePredicateFilter(colName, likeRegexp, expr.Operator == sqlparser.NotLikeStr) - pb.PrimitiveComposer.SetColVisited(colName, true) - return retVal, nil - } - operatorPredicate, preErr := relational.GetOperatorPredicate(expr.Operator) - - if preErr != nil { - return nil, preErr - } - - pb.PrimitiveComposer.SetColVisited(colName, true) - return relational.ConstructTablePredicateFilter(colName, resolved, operatorPredicate), nil -} - func (pb *standardPrimitiveGenerator) inferProviderForShow( node *sqlparser.Show, handlerCtx handler.HandlerContext) error { diff --git a/internal/stackql/primitivegenerator/select.go b/internal/stackql/primitivegenerator/select.go index e6031cc1c..a6f2a8bf1 100644 --- a/internal/stackql/primitivegenerator/select.go +++ b/internal/stackql/primitivegenerator/select.go @@ -4,17 +4,11 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/astvisit" - "github.com/stackql/stackql/internal/stackql/dependencyplanner" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/planbuilderinput" "github.com/stackql/stackql/internal/stackql/primitivebuilder" - "github.com/stackql/stackql/internal/stackql/symtab" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" "github.com/stackql/stackql/pkg/astformat" ) @@ -82,191 +76,8 @@ func (pb *standardPrimitiveGenerator) analyzeSelect(pbi planbuilderinput.PlanBui return pb.AnalyzeNop(pbi) } - selectMetadata, ok := annotatedAST.GetSelectMetadata(node) - if !ok { - return fmt.Errorf("could not obtain select metadata for select AST node") - } - - var pChild PrimitiveGenerator - var err error - - if err != nil { - return err - } - - tblz, hasTblz := selectMetadata.GetTableMap() - if !hasTblz { - return fmt.Errorf("select analysis: no table map present") - } - annotations, hasAnnotations := selectMetadata.GetAnnotations() - if !hasAnnotations { - return fmt.Errorf("select analysis not viable: no annotations present") - } - annotations.AssignParams() - existingParams := annotations.GetStringParams() - colRefs := pbi.GetColRefs() - // END_BLOCK ParameterHierarchy - - // BLOCK SequencingAccrual - dataFlows, ok := selectMetadata.GetOnConditionDataFlows() - if !ok { - return fmt.Errorf("could not obtain ON condition data flows for select AST node") - } - logging.GetLogger().Debugf("%v\n", dataFlows) - // END_BLOCK SequencingAccrual - - onConditionsToRewrite := selectMetadata.GetOnConditionsToRewrite() - - parserutil.NaiveRewriteComparisonExprs(onConditionsToRewrite) - - if err != nil { - return err - } - - for k, v := range tblz { - pb.PrimitiveComposer.SetTable(k, v) - } - - for i, fromExpr := range node.From { - var leafKey interface{} = i - switch from := fromExpr.(type) { //nolint:gocritic // TODO: review - case *sqlparser.AliasedTableExpr: - if from.As.GetRawVal() != "" { - leafKey = from.As.GetRawVal() - } - } - - leaf, err := pb.PrimitiveComposer.GetSymTab().NewLeaf(leafKey) - if err != nil { - return err - } - pChild = pb.AddChildPrimitiveGenerator(fromExpr, leaf) - - for _, selectExpr := range node.SelectExprs { - //nolint:gocritic // experimental - switch expr := selectExpr.(type) { - case *sqlparser.AliasedExpr: - alias := expr.As.GetRawVal() - if alias == "" { - continue - } - entry := symtab.NewSymTabEntry( - pb.PrimitiveComposer.GetDRMConfig().GetRelationalType("string"), - "", - alias, - ) - pb.PrimitiveComposer.SetSymbol(alias, entry) - } - } - - for _, tbl := range tblz { - err := pb.expandTable(tbl) - _, isIndirect := tbl.GetIndirect() - if err != nil && !isIndirect { - return err - } - } - } - - // BLOCK REWRITE_WHERE - // TODO: fix this hack - // might make sense to implement an "all in one" - // query rewrite as an AST visitor. - var rewrittenWhere *sqlparser.Where - var paramsPresent []string - if len(node.From) == 1 { - switch ft := node.From[0].(type) { - case *sqlparser.ExecSubquery: - logging.GetLogger().Infoln(fmt.Sprintf("%v", ft)) - default: - rewrittenWhere, paramsPresent, err = pb.analyzeWhere(node.Where, existingParams) - if err != nil { - return err - } - pb.PrimitiveComposer.SetWhere(rewrittenWhere) - } - } - logging.GetLogger().Debugf("len(paramsPresent) = %d\n", len(paramsPresent)) - // END_BLOCK REWRITE_WHERE - - isSimpleFrom := parserutil.IsFromExprSimple(node.From) - - if len(node.From) >= 1 && isSimpleFrom { - switch node.From[0].(type) { - case *sqlparser.JoinTableExpr, *sqlparser.AliasedTableExpr: - tcc := pbi.GetTxnCtrlCtrs() - tccAheadOfTime := pbi.IsTccSetAheadOfTime() - dp, err := dependencyplanner.NewStandardDependencyPlanner( - annotatedAST, - handlerCtx, - dataFlows, - colRefs, - rewrittenWhere, - node, - tblz, - pb.PrimitiveComposer, - tcc, - tccAheadOfTime, - ) - if err != nil { - return err - } - dp = dp.WithPrepStmtOffset(pb.prepStmtOffset) - dp = dp.WithElideRead(pb.IsElideRead()) - err = dp.Plan() - if err != nil { - return err - } - bld := dp.GetBldr() - if pb.PrimitiveComposer.IsIndirect() { - pb.SetIndirectCreateTailBuilder([]primitivebuilder.Builder{bld}) - } - selCtx := dp.GetSelectCtx() - pChild.GetPrimitiveComposer().SetBuilder(bld) - pb.PrimitiveComposer.SetSelectPreparedStatementCtx(selCtx) - return nil - } - } - //nolint:gocritic // tactical - if len(node.From) == 1 { - switch ft := node.From[0].(type) { - case *sqlparser.ExecSubquery: - cols, err := parserutil.ExtractSelectColumnNames(node, handlerCtx.GetASTFormatter()) - if err != nil { - return err - } - tbl, err := pChild.AnalyzeUnaryExec(pbi, handlerCtx, ft.Exec, node, cols) - if err != nil { - return err - } - insertionContainer, err := tableinsertioncontainer.NewTableInsertionContainer( - tbl, - handlerCtx.GetSQLEngine(), - handlerCtx.GetTxnCounterMgr()) - if err != nil { - return err - } - selIndirect, indirectErr := astindirect.NewParserSelectIndirect( - node, pChild.GetPrimitiveComposer().GetSelectPreparedStatementCtx()) - if indirectErr != nil { - return indirectErr - } - annotatedAST.SetSelectIndirect(node, selIndirect) - bldrInput := builder_input.NewBuilderInput( - pChild.GetPrimitiveComposer().GetGraphHolder(), - handlerCtx, - tbl, - ) - bldrInput.SetTxnCtrlCtrs(pChild.GetPrimitiveComposer().GetTxnCtrlCtrs()) - bldrInput.SetTableInsertionContainer(insertionContainer) - pChild.GetPrimitiveComposer().SetBuilder( - primitivebuilder.NewSingleAcquireAndSelect( - bldrInput, - pChild.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), - pChild.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), - nil)) - return nil - } + if table, err := parserutil.ExtractSingleTableFromTableExprs(node.From); err == nil { + return fmt.Errorf("could not locate table '%s'", table.Name.GetRawVal()) } - return fmt.Errorf("cannot process cartesian join select just yet") + return fmt.Errorf("SELECT must be planned through omnisdk or the SQL backend") } diff --git a/internal/stackql/primitivegenerator/statement_analyzer.go b/internal/stackql/primitivegenerator/statement_analyzer.go index 4f1edaac8..51d6d2fb0 100644 --- a/internal/stackql/primitivegenerator/statement_analyzer.go +++ b/internal/stackql/primitivegenerator/statement_analyzer.go @@ -1,37 +1,20 @@ package primitivegenerator import ( - "encoding/json" "fmt" - "math" - "regexp" - "strconv" "strings" - "time" - "github.com/getkin/kin-openapi/openapi3" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - anysdk_internaldto "github.com/stackql/any-sdk/pkg/internaldto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/astvisit" + "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/intrinsic" "github.com/stackql/stackql/internal/stackql/iqlerror" - "github.com/stackql/stackql/internal/stackql/iqlutil" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/planbuilderinput" - "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/primitivebuilder" - "github.com/stackql/stackql/internal/stackql/provider" - "github.com/stackql/stackql/internal/stackql/relational" - "github.com/stackql/stackql/internal/stackql/suffix" - "github.com/stackql/stackql/internal/stackql/symtab" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/internal/stackql/taxonomy" "github.com/stackql/stackql/internal/stackql/util" @@ -40,11 +23,6 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" ) -var ( - synonymJSONRegexp *regexp.Regexp = regexp.MustCompile(`^application/[\S]*json[\S]*$`) - synonymXMLRegexp *regexp.Regexp = regexp.MustCompile(`^(?:application|text)/[\S]*xml[\S]*$`) -) - //nolint:funlen // this is unavoidable func (pb *standardPrimitiveGenerator) AnalyzeStatement( pbi planbuilderinput.PlanBuilderInput, @@ -53,9 +31,9 @@ func (pb *standardPrimitiveGenerator) AnalyzeStatement( statement := pbi.GetStatement() switch stmt := statement.(type) { case *sqlparser.Auth: - return pb.analyzeAuth(pbi) + return nil case *sqlparser.AuthRevoke: - return pb.analyzeAuthRevoke(pbi) + return nil case *sqlparser.Begin: return iqlerror.GetStatementNotSupportedError("TRANSACTION: BEGIN") case *sqlparser.Commit: @@ -68,8 +46,6 @@ func (pb *standardPrimitiveGenerator) AnalyzeStatement( return pb.analyzeDelete(pbi) case *sqlparser.DescribeTable: return pb.analyzeDescribe(pbi) - case *sqlparser.Exec: - return pb.analyzeExec(pbi) case *sqlparser.Explain: return fmt.Errorf("direct access to %s analysis not supported", "EXPLAIN") case *sqlparser.Insert: @@ -91,7 +67,7 @@ func (pb *standardPrimitiveGenerator) AnalyzeStatement( case *sqlparser.Show: return pb.analyzeShow(pbi) case *sqlparser.Sleep: - return pb.analyzeSleep(pbi) + return nil case *sqlparser.SRollback: return iqlerror.GetStatementNotSupportedError("TRANSACTION: SROLLBACK") case *sqlparser.Release: @@ -243,298 +219,6 @@ func (pb *standardPrimitiveGenerator) AnalyzeSelectStatement( return nil } -func (pb *standardPrimitiveGenerator) analyzeAuth( - pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - authNode, ok := pbi.GetAuth() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Auth", pbi.GetStatement()) - } - provider, pErr := handlerCtx.GetProvider(authNode.Provider) - if pErr != nil { - return pErr - } - pb.PrimitiveComposer.SetProvider(provider) - return nil -} - -func (pb *standardPrimitiveGenerator) analyzeAuthRevoke( - pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - authNode, ok := pbi.GetAuthRevoke() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required AuthRevoke", pbi.GetStatement()) - } - authCtx, authErr := handlerCtx.GetAuthContext(authNode.Provider) - if authErr != nil { - return authErr - } - switch strings.ToLower(authCtx.Type) { - case dto.AuthServiceAccountStr, dto.AuthInteractiveStr: - return nil - } - //nolint:stylecheck // prescribed - return fmt.Errorf(`Auth revoke for Google Failed; improper auth method: "%s" specified`, authCtx.Type) -} - -func checkResource( - handlerCtx handler.HandlerContext, - prov provider.IProvider, - service string, - resource string, -) (formulation.Resource, error) { - return prov.GetResource(service, resource, handlerCtx.GetRuntimeContext()) -} - -func (pb *standardPrimitiveGenerator) assembleResources( - handlerCtx handler.HandlerContext, - prov provider.IProvider, - service string, -) (map[string]formulation.Resource, error) { - rm, err := prov.GetResourcesMap(service, handlerCtx.GetRuntimeContext()) - if err != nil { - return nil, err - } - return rm, err -} - -func (pb *standardPrimitiveGenerator) analyzeShowFilter(node *sqlparser.Show, table formulation.ITable) error { - showFilter := node.ShowTablesOpt.Filter - if showFilter == nil { - return nil - } - if showFilter.Like != "" { - likeRegexp, err := regexp.Compile(iqlutil.TranslateLikeToRegexPattern(showFilter.Like)) - if err != nil { - return fmt.Errorf("cannot compile like string '%s': %w", showFilter.Like, err) - } - tableFilter := pb.PrimitiveComposer.GetTableFilter() - for _, col := range pb.PrimitiveComposer.GetLikeAbleColumns() { - tableFilter = relational.OrTableFilters(tableFilter, relational.ConstructLikePredicateFilter(col, likeRegexp, false)) - } - pb.PrimitiveComposer.SetTableFilter(relational.OrTableFilters(pb.PrimitiveComposer.GetTableFilter(), tableFilter)) - } else if showFilter.Filter != nil { - tableFilter, err := pb.traverseShowFilter(table, node, showFilter.Filter) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTableFilter(tableFilter) - } - return nil -} - -func (pb *standardPrimitiveGenerator) traverseShowFilter( - table formulation.ITable, - node *sqlparser.Show, - filter sqlparser.Expr, -) (func(formulation.ITable) (formulation.ITable, error), error) { - switch filter := filter.(type) { - case *sqlparser.ComparisonExpr: - return pb.comparisonExprToFilterFunc(table, node, filter) - case *sqlparser.AndExpr: - logging.GetLogger().Infoln("complex AND expr detected") - lhs, lhErr := pb.traverseShowFilter(table, node, filter.Left) - rhs, rhErr := pb.traverseShowFilter(table, node, filter.Right) - if lhErr != nil { - return nil, lhErr - } - if rhErr != nil { - return nil, rhErr - } - return relational.AndTableFilters(lhs, rhs), nil - case *sqlparser.OrExpr: - logging.GetLogger().Infoln("complex OR expr detected") - lhs, lhErr := pb.traverseShowFilter(table, node, filter.Left) - rhs, rhErr := pb.traverseShowFilter(table, node, filter.Right) - if lhErr != nil { - return nil, lhErr - } - if rhErr != nil { - return nil, rhErr - } - return relational.OrTableFilters(lhs, rhs), nil - case *sqlparser.FuncExpr: - return nil, fmt.Errorf("unsupported constraint in anysdk filter: %v", sqlparser.String(filter)) - default: - return nil, fmt.Errorf("unsupported constraint in anysdk filter: %v", sqlparser.String(filter)) - } -} - -func (pb *standardPrimitiveGenerator) traverseWhereFilter( - node sqlparser.SQLNode, - requiredParameters, - optionalParameters suffix.ParameterSuffixMap, -) (sqlparser.Expr, []string, error) { - switch node := node.(type) { - case *sqlparser.ComparisonExpr: - exp, cn, err := pb.whereComparisonExprCopyAndReWrite(node, requiredParameters, optionalParameters) - return exp, []string{cn}, err - case *sqlparser.AndExpr: - logging.GetLogger().Infoln("complex AND expr detected") - lhs, lParams, lhErr := pb.traverseWhereFilter(node.Left, requiredParameters, optionalParameters) - rhs, rParams, rhErr := pb.traverseWhereFilter(node.Right, requiredParameters, optionalParameters) - if lhErr != nil { - return nil, nil, lhErr - } - if rhErr != nil { - return nil, nil, rhErr - } - lParams = append(lParams, rParams...) - return &sqlparser.AndExpr{Left: lhs, Right: rhs}, lParams, nil - case *sqlparser.OrExpr: - logging.GetLogger().Infoln("complex OR expr detected") - lhs, lParams, lhErr := pb.traverseWhereFilter(node.Left, requiredParameters, optionalParameters) - rhs, rParams, rhErr := pb.traverseWhereFilter(node.Right, requiredParameters, optionalParameters) - if lhErr != nil { - return nil, nil, lhErr - } - if rhErr != nil { - return nil, nil, rhErr - } - lParams = append(lParams, rParams...) - return &sqlparser.OrExpr{Left: lhs, Right: rhs}, lParams, nil - case *sqlparser.FuncExpr: - return nil, nil, fmt.Errorf("unsupported constraint in anysdk filter: %v", sqlparser.String(node)) - case *sqlparser.IsExpr: - return &sqlparser.IsExpr{ - Operator: node.Operator, - Expr: node.Expr, - }, nil, nil - default: - return nil, nil, fmt.Errorf("unsupported constraint in anysdk filter: %v", sqlparser.String(node)) - } -} - -func (pb *standardPrimitiveGenerator) whereComparisonExprCopyAndReWrite( - expr *sqlparser.ComparisonExpr, - requiredParameters, - optionalParameters suffix.ParameterSuffixMap, -) (sqlparser.Expr, string, error) { - if parserutil.IsScalarComparison(expr) { - return expr, "", nil - } - qualifiedName, ok := expr.Left.(*sqlparser.ColName) - if !ok { - if qualifiedLHSName, lhsOk := expr.Left.(*sqlparser.FuncExpr); lhsOk { - funcName := strings.ToLower(qualifiedLHSName.Name.GetRawVal()) - logging.GetLogger().Infoln(fmt.Sprintf("funcName = %s", funcName)) - return expr, "", nil - } else if qualifiedRHSName, rhsOk := expr.Right.(*sqlparser.FuncExpr); rhsOk { - funcName := strings.ToLower(qualifiedRHSName.Name.GetRawVal()) - logging.GetLogger().Infoln(fmt.Sprintf("funcName = %s", funcName)) - return expr, "", nil - } - return nil, "", fmt.Errorf("unexpected: %v", sqlparser.String(expr)) - } - colName := internaldto.GeneratePutativelyUniqueColumnID(qualifiedName.Qualifier, qualifiedName.Name.GetRawVal()) - symTabEntry, symTabErr := pb.PrimitiveComposer.GetSymbol(colName) - _, requiredParamPresent := requiredParameters.Get(colName) - _, optionalParamPresent := optionalParameters.Get(colName) - logging.GetLogger().Infoln(fmt.Sprintf("symTabEntry = %v", symTabEntry)) - containsSQLDataSource := pb.GetPrimitiveComposer().ContainsSQLDataSource() - if !containsSQLDataSource && symTabErr != nil && !(requiredParamPresent || optionalParamPresent) { - return nil, colName, symTabErr - } - if requiredParamPresent { - requiredParameters.Delete(colName) - } - if optionalParamPresent { - optionalParameters.Delete(colName) - } - if containsSQLDataSource { - return expr, colName, nil - } - if symTabErr == nil && symTabEntry.In != "server" { - if !(requiredParamPresent || optionalParamPresent) { - rightExpr := expr.Right - if boolVal, boolOk := expr.Right.(sqlparser.BoolVal); boolOk { - // Provider-backed rows are often materialized as text, so normalize bool literals. - rightExpr = &sqlparser.SQLVal{Type: sqlparser.StrVal, Val: []byte(strconv.FormatBool(bool(boolVal)))} - } - return &sqlparser.ComparisonExpr{ - Left: expr.Left, - Right: rightExpr, - Operator: expr.Operator, - Escape: expr.Escape, - }, colName, nil - } - } - rewrittenOperator := expr.Operator - if strings.ToUpper(expr.Operator) == "IN" { - rewrittenOperator = "=" - } - return &sqlparser.ComparisonExpr{ - Left: &sqlparser.SQLVal{Type: sqlparser.IntVal, Val: []byte("1")}, - Right: &sqlparser.SQLVal{Type: sqlparser.IntVal, Val: []byte("1")}, - Operator: rewrittenOperator, - Escape: expr.Escape, - }, colName, nil -} - -func (pb *standardPrimitiveGenerator) analyzeWhere( - where *sqlparser.Where, - existingParams map[string]interface{}, -) (*sqlparser.Where, []string, error) { - var retVal sqlparser.Expr - var paramsSupplied []string - tableParameterCollection, err := pb.PrimitiveComposer.AssignParameters() - if err != nil { - return nil, paramsSupplied, err - } - optionalParameters := tableParameterCollection.GetOptionalParams() - requiredParameters := tableParameterCollection.GetRequiredParams() - remainingRequiredParameters := tableParameterCollection.GetRemainingRequiredParams() - if where != nil { - retVal, paramsSupplied, err = pb.traverseWhereFilter(where.Expr, requiredParameters, optionalParameters) - if err != nil { - return nil, paramsSupplied, err - } - } - - for k := range existingParams { - remainingRequiredParameters.Delete(k) - } - - // TODO: consume parent parameters for any shortfall in required params - // TODO: same, for optional params - - isIndirect := pb.PrimitiveComposer.IsIndirect() - if remainingRequiredParameters.Size() > 0 && !isIndirect { - if where == nil { - return nil, paramsSupplied, - fmt.Errorf("WHERE clause not supplied, run DESCRIBE EXTENDED for the resource to see required parameters") - } - var keys []string - for k := range remainingRequiredParameters.GetAll() { - keys = append(keys, k) - } - return nil, paramsSupplied, - fmt.Errorf("query cannot be executed, missing required parameters: { %s }", strings.Join(keys, ", ")) - } - if where == nil { - return nil, paramsSupplied, nil - } - return &sqlparser.Where{Type: where.Type, Expr: retVal}, paramsSupplied, nil -} - -func extractVarDefFromExec( - node *sqlparser.Exec, - method formulation.OperationStore, - argName string, -) (*sqlparser.ExecVarDef, error) { - for _, varDef := range node.ExecVarDefs { - rawName := varDef.ColIdent.GetRawVal() - if rawName == argName { - return &varDef, nil - } - // A snake alias resolves to its wire parameter (any-sdk #131). - if p, ok := method.GetParameter(rawName); ok && p.GetName() == argName { - return &varDef, nil - } - } - return nil, fmt.Errorf("could not find variable '%s'", argName) -} - func (pb *standardPrimitiveGenerator) parseComments(comments sqlparser.Comments) { if comments != nil { commentDirectives := sqlparser.ExtractCommentDirectives(comments) @@ -564,123 +248,6 @@ func (pb *standardPrimitiveGenerator) persistHerarchyToBuilder( } //nolint:funlen,gocognit // TODO: refactor -func (pb *standardPrimitiveGenerator) AnalyzeUnaryExec( - pbi planbuilderinput.PlanBuilderInput, - handlerCtx handler.HandlerContext, - node *sqlparser.Exec, - selectNode *sqlparser.Select, - cols []parserutil.ColumnHandle, -) (tablemetadata.ExtendedTableMetadata, error) { - isAwait := pb.hasAwaitComment(node.Comments) - err := pb.inferHeirarchyAndPersist(handlerCtx, node, nil, false, isAwait) - if err != nil { - return nil, err - } - pb.parseComments(node.Comments) - - meta, err := pb.PrimitiveComposer.GetTable(node) - if err != nil { - return nil, err - } - - method, err := meta.GetMethod() - if err != nil { - return nil, err - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return nil, fmt.Errorf("method %s is not awaitable", method.GetName()) - } - - requiredParams := method.GetRequiredParameters() - - colz, err := parserutil.GetColumnUsageTypesForExec(node) - if err != nil { - return nil, err - } - usageErr := parserutil.CheckColUsagesAgainstTable(colz, method) - if usageErr != nil { - return nil, usageErr - } - for k, param := range requiredParams { - logging.GetLogger().Debugln(fmt.Sprintf("param = %v", param)) - _, err = extractVarDefFromExec(node, method, k) - if err != nil { - return nil, fmt.Errorf("required param not supplied for exec: %w", err) - } - } - prov, err := meta.GetProvider() - if err != nil { - return nil, err - } - svcStr, err := meta.GetServiceStr() - if err != nil { - return nil, err - } - svc, err := meta.GetService() - if err != nil { - return nil, err - } - rStr, err := meta.GetResourceStr() - if err != nil { - return nil, err - } - logging.GetLogger().Infoln( - fmt.Sprintf("provider = '%s', service = '%s', resource = '%s'", - prov.GetProviderString(), svcStr, rStr)) - requestSchema, err := method.GetRequestBodySchema() - // requestSchema, err := prov.GetObjectSchema(svcStr, rStr, method.Request.BodyMediaType) - req, reqExists := method.GetRequest() - if err != nil && reqExists { - return nil, err - } - var execPayload anysdk_internaldto.ExecPayload - if node.OptExecPayload != nil { - mediaType := "application/json" //nolint:goconst // acceptable for now - if reqExists && req.GetBodyMediaType() != "" { - mediaType = req.GetBodyMediaType() - } - execPayload, err = pb.parseExecPayload(node.OptExecPayload, mediaType) - if err != nil { - return nil, err - } - err = pb.analyzeSchemaVsMap(handlerCtx, requestSchema, execPayload.GetPayloadMap(), method) - if err != nil { - return nil, err - } - } - rsc, err := meta.GetResource() - if err != nil { - return nil, err - } - err = pb.buildRequestContext(node, meta, formulation.NewExecContext(execPayload, rsc), nil) - if err != nil { - return nil, err - } - pb.PrimitiveComposer.SetTable(node, meta) - - // parse response with SQL - if method.IsNullary() && !pb.PrimitiveComposer.IsAwait() { - return meta, nil - } - // TODO: columns in and replace hadnrolled analysis - analysisInput := formulation.NewMethodAnalysisInput( - method, - svc, - true, - []formulation.ColumnDescriptor{}, - false, - ) - analyser := formulation.NewMethodAnalyzer() - methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) - if analysisErr != nil { - return meta, analysisErr - } - if selectNode != nil { - return meta, pb.analyzeUnarySelection(pbi, handlerCtx, selectNode, selectNode.Where, meta, cols, methodAnalysisOutput) - } - return meta, pb.analyzeUnarySelection(pbi, handlerCtx, node, nil, meta, cols, methodAnalysisOutput) -} func (pb *standardPrimitiveGenerator) AnalyzeNop( pbi planbuilderinput.PlanBuilderInput) error { @@ -717,215 +284,10 @@ func (pb *standardPrimitiveGenerator) AnalyzeExplain( return nil } -func (pb *standardPrimitiveGenerator) analyzeExec(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - annotatedAST := pbi.GetAnnotatedAST() - node, ok := pbi.GetExec() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Exec", pbi.GetStatement()) - } - tbl, err := pb.AnalyzeUnaryExec(pbi, handlerCtx, node, nil, nil) - if err != nil { - return err - } - insertionContainer, err := tableinsertioncontainer.NewTableInsertionContainer( - tbl, - handlerCtx.GetSQLEngine(), - handlerCtx.GetTxnCounterMgr()) - if err != nil { - return err - } - m, err := tbl.GetMethod() - if err != nil { - return err - } - if m.IsNullary() && !pb.PrimitiveComposer.IsAwait() { - bldrInput := builder_input.NewBuilderInput( - pb.PrimitiveComposer.GetGraphHolder(), - handlerCtx, - tbl, - ) - bldrInput.SetIsAwait(false) // returning hardcoded to false for now - pb.PrimitiveComposer.SetBuilder( - primitivebuilder.NewSingleSelectAcquire( - pb.PrimitiveComposer.GetGraphHolder(), - handlerCtx, - insertionContainer, - pb.PrimitiveComposer.GetInsertPreparedStatementCtx(), - nil, - nil, - bldrInput, - ), - ) - return nil - } - selIndirect, indirectErr := astindirect.NewParserExecIndirect( - node, pb.PrimitiveComposer.GetSelectPreparedStatementCtx()) - if indirectErr != nil { - return indirectErr - } - annotatedAST.SetExecIndirect(node, selIndirect) - bldrInput := builder_input.NewBuilderInput( - pb.PrimitiveComposer.GetGraphHolder(), - handlerCtx, - tbl, - ) - bldrInput.SetTxnCtrlCtrs(pb.PrimitiveComposer.GetTxnCtrlCtrs()) - bldrInput.SetTableInsertionContainer(insertionContainer) - pb.PrimitiveComposer.SetBuilder( - primitivebuilder.NewSingleAcquireAndSelect( - bldrInput, - pb.PrimitiveComposer.GetInsertPreparedStatementCtx(), - pb.PrimitiveComposer.GetSelectPreparedStatementCtx(), - nil, - )) - return nil -} - -func (pb *standardPrimitiveGenerator) sanitiseMediaType(mediaType string) string { - if synonymJSONRegexp.MatchString(mediaType) { - return "application/json" - } - if synonymXMLRegexp.MatchString(mediaType) { - return "application/xml" - } - return mediaType -} - -func (pb *standardPrimitiveGenerator) parseExecPayload( - node *sqlparser.ExecVarDef, - payloadType string, -) (anysdk_internaldto.ExecPayload, error) { - var b []byte - m := make(map[string][]string) - var pm map[string]interface{} - payloadType = pb.sanitiseMediaType(payloadType) - switch val := node.Val.(type) { - case *sqlparser.SQLVal: - b = val.Val - default: - return nil, fmt.Errorf("payload map of SQL type = '%T' not allowed", val) - } - switch payloadType { - case constants.JSONStr, "application/json": - m["Content-Type"] = []string{"application/json"} - err := json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("payload map of declared type = '%T' not allowed", payloadType) - } - return anysdk_internaldto.NewExecPayload( - b, - m, - pm, - ), nil -} - // acceptsJSONNumber: JSON numbers unmarshal as float64, so an integral value // also satisfies an integer property (issue #725). -func acceptsJSONNumber(ss formulation.Schema, val float64) bool { - return ss.IsFloat() || (ss.IsIntegral() && val == math.Trunc(val)) -} //nolint:funlen,unparam,gocognit // TODO: refactor -func (pb *standardPrimitiveGenerator) analyzeSchemaVsMap( - handlerCtx handler.HandlerContext, - schema formulation.Schema, - payload map[string]interface{}, - method formulation.OperationStore, -) error { - if schema == nil { - return fmt.Errorf("method '%s' has no request body schema", method.GetName()) - } - requiredElements := make(map[string]bool) - schemas, err := schema.GetProperties() - if err != nil { - return err - } - for k := range schemas { - if schema.IsRequired(k) { - requiredElements[k] = true - } - } - for k, v := range payload { - ss, propertyExists := schema.GetProperty(k) - if !propertyExists { - return fmt.Errorf("schema does not possess payload key '%s'", k) - } - switch val := v.(type) { - case map[string]interface{}: - delete(requiredElements, k) - err = pb.analyzeSchemaVsMap(handlerCtx, ss, val, method) - if err != nil { - return err - } - case []interface{}: - subSchema, sErr := schema.GetPropertySchema(k) - if sErr != nil { - return sErr - } - arraySchema, itemsErr := subSchema.GetItemsSchema() - if itemsErr != nil { - return itemsErr - } - delete(requiredElements, k) - if len(val) > 0 && val[0] != nil { - switch item := val[0].(type) { - case map[string]interface{}: - err = pb.analyzeSchemaVsMap(handlerCtx, arraySchema, item, method) - if err != nil { - return err - } - case string: - if arraySchema.GetType() != "string" { - return fmt.Errorf( - "array at key '%s' expected to contain elemenst of type 'string' but instead they are type '%T'", - k, item) - } - default: - return fmt.Errorf("array at key '%s' does not contain recognisable type '%T'", k, item) - } - } - case string: - if ss.GetType() != "string" { - return fmt.Errorf("key '%s' expected to contain element of type 'string' but instead it is type '%T'", k, val) - } - delete(requiredElements, k) - case int: - if ss.IsIntegral() { - delete(requiredElements, k) - continue - } - return fmt.Errorf("key '%s' expected to contain element of type 'int' but instead it is type '%T'", k, val) - case bool: - if ss.IsBoolean() { - delete(requiredElements, k) - continue - } - return fmt.Errorf("key '%s' expected to contain element of type 'bool' but instead it is type '%T'", k, val) - case float64: - if acceptsJSONNumber(ss, val) { - delete(requiredElements, k) - continue - } - return fmt.Errorf("key '%s' expected to contain element of type 'float64' but instead it is type '%T'", k, val) - default: - return fmt.Errorf("key '%s' of type '%T' not currently supported", k, val) - } - } - if len(requiredElements) != 0 { - var missingKeys []string - for k := range requiredElements { - missingKeys = append(missingKeys, k) - } - return fmt.Errorf( - "required elements not included in suplied object; the following keys are missing: %s", - strings.Join(missingKeys, ", ")) - } - return nil -} func (pb *standardPrimitiveGenerator) AnalyzePGInternal( pbi planbuilderinput.PlanBuilderInput) error { @@ -968,201 +330,43 @@ func (pb *standardPrimitiveGenerator) AnalyzePGInternal( } //nolint:funlen,gocognit // TODO: refactor -func (pb *standardPrimitiveGenerator) expandTable( - tbl tablemetadata.ExtendedTableMetadata) error { - if viewIndirect, isView := tbl.GetIndirect(); isView { - viewAST := viewIndirect.GetSelectAST() - - mergeErr := pb.PrimitiveComposer.MergeSymTab(viewIndirect.GetUnderlyingSymTab(), tbl.GetAlias()) - if mergeErr != nil { - return mergeErr - } - logging.GetLogger().Debugf("viewAST = %v\n", viewAST) - return nil - } - if sqlDataSource, isSQLDataSource := tbl.GetSQLDataSource(); isSQLDataSource { - logging.GetLogger().Debugf("sqlDataSource = %v\n", sqlDataSource) - return nil +//nolint:gocognit,funlen,gocyclo,cyclop // TODO: review +func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBuilderInput) error { + handlerCtx := pbi.GetHandlerCtx() + annotatedAST := pbi.GetAnnotatedAST() + node, ok := pbi.GetInsert() + if !ok { + return fmt.Errorf("could not cast node of type '%T' to required Insert", pbi.GetStatement()) } - // TODO: encapsulate the mapping of openapi schemas to symbol table entries. - // - This operates atop DRM. - svc, err := tbl.GetService() + isAwait := pb.hasAwaitComment(node.Comments) + err := pb.inferHeirarchyAndPersist(handlerCtx, node, pbi.GetPlaceholderParams(), false, isAwait) if err != nil { return err } - var availableServers openapi3.Servers - var availableServersDoExist bool - method, _ := tbl.GetMethod() - if method != nil { - srvs, ok := method.GetServers() - if ok && len(srvs) > 0 { - availableServers = srvs - availableServersDoExist = true - } - } - if len(availableServers) == 0 { - availableServers, availableServersDoExist = svc.GetServers() - } - if availableServersDoExist { - for _, sv := range availableServers { - for k := range sv.Variables { - colEntry := symtab.NewSymTabEntry( - pb.PrimitiveComposer.GetDRMConfig().GetRelationalType("string"), - "", - "server", - ) - uid := fmt.Sprintf("%s.%s", tbl.GetUniqueID(), k) - pb.PrimitiveComposer.SetSymbol(uid, colEntry) //nolint:errcheck // TODO: review - } - break //nolint:staticcheck // TODO: review - } - } - responseSchema, err := tbl.GetSelectableObjectSchema() + tbl, err := pb.PrimitiveComposer.GetTable(node) if err != nil { return err } - cols, err := responseSchema.GetProperties() + pb.PrimitiveComposer.SetTable(node, tbl) + insertValOnlyRows, nonValCols, err := parserutil.ExtractInsertValColumnsPlusPlaceHolders(node) if err != nil { return err } - if len(cols) == 0 { - cols = formulation.Schemas{formulation.AnonymousColumnName: responseSchema} - } - for colName, colSchema := range cols { - if colSchema == nil { - return fmt.Errorf("could not infer column information") - } - colEntry := symtab.NewSymTabEntry( - pb.PrimitiveComposer.GetDRMConfig().GetRelationalType(colSchema.GetType()), - colSchema, - "", - ) - uid := fmt.Sprintf("%s.%s", tbl.GetUniqueID(), colName) - pb.PrimitiveComposer.SetSymbol(uid, colEntry) //nolint:errcheck // TODO: review - } - // Register the display column names alongside the raw property keys. - // Under config.snake_case_aliases the tabulation's display names are the - // snake aliases (any-sdk owns that translation), so WHERE / projection - // symbols resolve for either form; without aliasing this re-registers - // the same names and is a no-op. - if tabulation := responseSchema.Tabulate(false, ""); tabulation != nil { - for _, col := range tabulation.GetColumns() { - colSchema := col.GetSchema() - if colSchema == nil { - continue + pb.PrimitiveComposer.SetInsertValOnlyRows(insertValOnlyRows) + if nonValCols > 0 { + switch rowsNode := node.Rows.(type) { + case *sqlparser.Select: + for k, v := range insertValOnlyRows { + row := v + maxKey := util.MaxMapKey(row) + for i := 0; i < nonValCols; i++ { + row[maxKey+i+1] = "placeholder" + } + insertValOnlyRows[k] = row } - colEntry := symtab.NewSymTabEntry( - pb.PrimitiveComposer.GetDRMConfig().GetRelationalType(colSchema.GetType()), - colSchema, - "", - ) - uid := fmt.Sprintf("%s.%s", tbl.GetUniqueID(), col.GetName()) - pb.PrimitiveComposer.SetSymbol(uid, colEntry) //nolint:errcheck // duplicate registration is benign - } - } - return nil -} - -func (pb *standardPrimitiveGenerator) buildRequestContext( - node sqlparser.SQLNode, - meta tablemetadata.ExtendedTableMetadata, - execContext formulation.ExecContext, - rowsToInsert map[int]map[int]interface{}, -) error { - paramMapArray, paramErr := util.ExtractSQLNodeParams(node, rowsToInsert) - if paramErr != nil { - return paramErr - } - return pb.buildRequestContextFromMapArray( - node, - meta, - execContext, - paramMapArray, - ) -} - -func (pb *standardPrimitiveGenerator) buildRequestContextFromMapArray( - node sqlparser.SQLNode, //nolint:revive,unparam // TODO: review - meta tablemetadata.ExtendedTableMetadata, - execContext formulation.ExecContext, - paramMapArray map[int]map[string]interface{}, -) error { - m, err := meta.GetMethod() - if err != nil { - return err - } - prov, err := meta.GetProvider() - if err != nil { - return err - } - svc, err := meta.GetService() - if err != nil { - return err - } - pr, prErr := prov.GetProvider() - if prErr != nil { - return prErr - } - meta.WithGetHTTPArmoury( - func() (formulation.HTTPArmoury, error) { - httpPreparator := formulation.NewHTTPPreparator( - pr, - svc, - m, - paramMapArray, - nil, - execContext, - logging.GetLogger(), - ) - httpArmoury, httpErr := httpPreparator.BuildHTTPRequestCtx( - formulation.NewHTTPPreparatorConfig(false), - ) - if httpErr != nil { - return nil, httpErr - } - return httpArmoury, nil - }, - ) - return err -} - -//nolint:gocognit,funlen,gocyclo,cyclop // TODO: review -func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - annotatedAST := pbi.GetAnnotatedAST() - node, ok := pbi.GetInsert() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Insert", pbi.GetStatement()) - } - isAwait := pb.hasAwaitComment(node.Comments) - err := pb.inferHeirarchyAndPersist(handlerCtx, node, pbi.GetPlaceholderParams(), false, isAwait) - if err != nil { - return err - } - tbl, err := pb.PrimitiveComposer.GetTable(node) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTable(node, tbl) - insertValOnlyRows, nonValCols, err := parserutil.ExtractInsertValColumnsPlusPlaceHolders(node) - if err != nil { - return err - } - pb.PrimitiveComposer.SetInsertValOnlyRows(insertValOnlyRows) - if nonValCols > 0 { - switch rowsNode := node.Rows.(type) { - case *sqlparser.Select: - for k, v := range insertValOnlyRows { - row := v - maxKey := util.MaxMapKey(row) - for i := 0; i < nonValCols; i++ { - row[maxKey+i+1] = "placeholder" - } - insertValOnlyRows[k] = row - } - default: - return fmt.Errorf("insert with rows of type '%T' not currently supported", rowsNode) + default: + return fmt.Errorf("insert with rows of type '%T' not currently supported", rowsNode) } } else { selQuery := strings.ReplaceAll(astformat.String(node.Rows, handlerCtx.GetASTFormatter()), "from \"dual\"", "") @@ -1188,100 +392,6 @@ func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBui } annotatedAST.SetInsertRowsIndirect(node, selectIndirect) } - isPhysicalTable := tbl.IsPhysicalTable() - if isPhysicalTable { - return nil - } - prov, err := tbl.GetProvider() - if err != nil { - return err - } - currentService, err := tbl.GetServiceStr() - if err != nil { - return err - } - currentResource, err := tbl.GetResourceStr() - if err != nil { - return err - } - - pb.parseComments(node.Comments) - - method, err := tbl.GetMethod() - if err != nil { - return err - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - - _, err = checkResource(handlerCtx, prov, currentService, currentResource) - if err != nil { - return err - } - pb.parseComments(node.Comments) - - if tbl.IsPhysicalTable() { - return nil - } - svc, err := tbl.GetService() - if err != nil { - return err - } - isOpenapi := svc.IsOpenapi() - if !isOpenapi { - err = pb.buildRequestContext(node, tbl, nil, insertValOnlyRows) - if err != nil { - return err - } - return nil - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - analysisInput := formulation.NewMethodAnalysisInput( - method, - svc, - true, - []formulation.ColumnDescriptor{}, - pb.PrimitiveComposer.IsAwait(), - ) - analyser := formulation.NewMethodAnalyzer() - // TODO: this ought to cater for async - methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) - if analysisErr != nil { - return analysisErr - } - err = pb.buildRequestContext(node, tbl, nil, insertValOnlyRows) - if err != nil { - return err - } - columnHandles := []parserutil.ColumnHandle{} - if len(node.SelectExprs) > 0 { - starColumns, _ := methodAnalysisOutput.GetOrderedStarColumnsNames() - columnHandles, err = parserutil.ExtractInsertReturningColumnNames(node, starColumns, handlerCtx.GetASTFormatter()) - if err != nil { - return err - } - } - err = pb.analyzeUnaryAction( - pbi, - handlerCtx, - node, - nil, - tbl, - columnHandles, - methodAnalysisOutput, - ) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTable(node, tbl) return nil } @@ -1304,99 +414,6 @@ func (pb *standardPrimitiveGenerator) AnalyzeUpdate(pbi planbuilderinput.PlanBui pb.PrimitiveComposer.SetTable(node, tbl) - isPhysicalTable := tbl.IsPhysicalTable() - if isPhysicalTable { - return nil - } - - pb.parseComments(node.Comments) - - method, err := tbl.GetMethod() - if err != nil { - return err - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - - if tbl.IsPhysicalTable() { - return nil - } - svc, err := tbl.GetService() - if err != nil { - return err - } - updateValOnlyRows, _, err := parserutil.ExtractUpdateValColumns(node) - if err != nil { - return err - } - firstRow := make(map[string]interface{}) - for k, v := range updateValOnlyRows { - firstRow[k.GetRawVal()] = v - } - updateValOnlyRowsMap := map[int]map[string]interface{}{0: firstRow} - isOpenapi := svc.IsOpenapi() - if !isOpenapi { - err = pb.buildRequestContextFromMapArray( - node, - tbl, - nil, - updateValOnlyRowsMap, - ) - if err != nil { - return err - } - return nil - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - analysisInput := formulation.NewMethodAnalysisInput( - method, - svc, - true, - []formulation.ColumnDescriptor{}, - pb.PrimitiveComposer.IsAwait(), - ) - analyser := formulation.NewMethodAnalyzer() - methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) - if analysisErr != nil { - return analysisErr - } - err = pb.buildRequestContextFromMapArray(node, tbl, nil, updateValOnlyRowsMap) - if err != nil { - return err - } - columnHandles := []parserutil.ColumnHandle{} - if len(node.SelectExprs) > 0 { - starColumns, _ := methodAnalysisOutput.GetOrderedStarColumnsNames() - columnHandles, err = parserutil.ExtractUpdateReturningColumnNames( - node, - starColumns, - handlerCtx.GetASTFormatter(), - ) - if err != nil { - return err - } - } - err = pb.analyzeUnaryAction( - pbi, - handlerCtx, - node, - nil, - tbl, - columnHandles, - methodAnalysisOutput, - ) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTable(node, tbl) return nil } @@ -1425,166 +442,15 @@ func (pb *standardPrimitiveGenerator) analyzeDelete( return fmt.Errorf("could not cast node of type '%T' to required Delete", pbi.GetStatement()) } pb.parseComments(node.Comments) - paramMap, paramMapExists := pbi.GetAnnotatedAST().GetWhereParamMapsEntry(node.Where) + paramMap, _ := pbi.GetAnnotatedAST().GetWhereParamMapsEntry(node.Where) isAwait := pb.hasAwaitComment(node.Comments) err := pb.inferHeirarchyAndPersist(handlerCtx, node, paramMap, false, isAwait) if err != nil { return err } - tbl, err := pb.PrimitiveComposer.GetTable(node) - if err != nil { - return err - } - - if tbl.IsPhysicalTable() { - return nil - } - - if !paramMapExists { - return fmt.Errorf("where parameters not found; should be anlaysed a priori") - } - method, err := tbl.GetMethod() - if err != nil { - return err - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - if pb.PrimitiveComposer.IsAwait() && len(node.SelectExprs) > 0 { - return fmt.Errorf("returning from asynchronous delete is disallowed") - } - svc, err := tbl.GetService() - if err != nil { - return err - } - - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { - return fmt.Errorf("method %s is not awaitable", method.GetName()) - } - analysisInput := formulation.NewMethodAnalysisInput( - method, - svc, - true, - []formulation.ColumnDescriptor{}, - pb.PrimitiveComposer.IsAwait(), - ) - analyser := formulation.NewMethodAnalyzer() - methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) - if analysisErr != nil { - return analysisErr - } - err = pb.buildRequestContext(node, tbl, nil, nil) - if err != nil { - return err - } - columnHandles := []parserutil.ColumnHandle{} - if len(node.SelectExprs) > 0 { - if pb.PrimitiveComposer.IsAwait() { - return fmt.Errorf("delete with returning not allowed in await mode") - } - starColumns, _ := methodAnalysisOutput.GetOrderedStarColumnsNames() - columnHandles, err = parserutil.ExtractDeleteReturningColumnNames( - node, - starColumns, - handlerCtx.GetASTFormatter(), - ) - if err != nil { - return err - } - } - err = pb.analyzeUnaryAction( - pbi, - handlerCtx, - node, - node.Where, - tbl, - columnHandles, - methodAnalysisOutput, - ) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTable(node, tbl) return nil } -func (pb *standardPrimitiveGenerator) analyzeDescribe(pbi planbuilderinput.PlanBuilderInput) error { - handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetDescribeTable() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Describe", pbi.GetStatement()) - } - var err error - // DESCRIBE has no await or comments at this time - err = pb.inferHeirarchyAndPersist(handlerCtx, node, nil, true, false) - if err != nil { - return err - } - tbl, err := pb.PrimitiveComposer.GetTable(node) - if err != nil { - return err - } - _, isView := tbl.GetView() - if isView { - return nil - } - prov, err := tbl.GetProvider() - if err != nil { - return err - } - currentService, err := tbl.GetServiceStr() - if err != nil { - return err - } - currentResource, err := tbl.GetResourceStr() - if err != nil { - return err - } - _, err = checkResource(handlerCtx, prov, currentService, currentResource) - if err != nil { - return err - } - return nil -} - -func (pb *standardPrimitiveGenerator) analyzeSleep(pbi planbuilderinput.PlanBuilderInput) error { - // handlerCtx := pbi.GetHandlerCtx() - node, ok := pbi.GetSleep() - if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Sleep", pbi.GetStatement()) - } - sleepDuration, err := parserutil.ExtractSleepDuration(node) - if err != nil { - return err - } - if sleepDuration <= 0 { - return fmt.Errorf("sleep duration %d not allowed, must be > 0", sleepDuration) - } - graph := pb.PrimitiveComposer.GetGraphHolder() - pb.PrimitiveComposer.SetRoot( - graph.CreatePrimitiveNode( - primitive.NewLocalPrimitive( - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { - time.Sleep(time.Duration(sleepDuration) * time.Millisecond) - return internaldto.NewExecutorOutput( - nil, nil, nil, - internaldto.NewBackendMessages( - []string{ - fmt.Sprintf("Success: slept for %d milliseconds", sleepDuration), - }, - ), nil) - }, - ), - ), - ) - return err -} - func (pb *standardPrimitiveGenerator) AnalyzeRegistry(pbi planbuilderinput.PlanBuilderInput) error { _, ok := pbi.GetRegistry() if !ok { @@ -1593,148 +459,40 @@ func (pb *standardPrimitiveGenerator) AnalyzeRegistry(pbi planbuilderinput.PlanB return nil } -//nolint:funlen,gocognit,gocyclo,cyclop,govet // TODO: review -func (pb *standardPrimitiveGenerator) analyzeShow( - pbi planbuilderinput.PlanBuilderInput) error { - var err error +func (pb *standardPrimitiveGenerator) analyzeShow(pbi planbuilderinput.PlanBuilderInput) error { handlerCtx := pbi.GetHandlerCtx() node, ok := pbi.GetShow() if !ok { - return fmt.Errorf("could not cast node of type '%T' to required Show", pbi.GetStatement()) + return fmt.Errorf("expected SHOW, got %T", pbi.GetStatement()) } - if sel, ok := planbuilderinput.IsPGSetupQuery(pbi); ok { - if sel != nil { - bldr := primitivebuilder.NewNativeSelect(pb.PrimitiveComposer.GetGraphHolder(), handlerCtx, sel) - pb.PrimitiveComposer.SetBuilder(bldr) - return nil + if selected, setup := planbuilderinput.IsPGSetupQuery(pbi); setup { + if selected == nil { + return pb.AnalyzeNop(pbi) } - return pb.AnalyzeNop(pbi) + pb.PrimitiveComposer.SetBuilder(primitivebuilder.NewNativeSelect( + pb.PrimitiveComposer.GetGraphHolder(), handlerCtx, selected)) + return nil } pb.parseComments(node.Comments) - err = pb.inferProviderForShow(node, handlerCtx) - if err != nil { - return err - } - nodeTypeUpperCase := strings.ToUpper(node.Type) - if pb.PrimitiveComposer.GetProvider() != nil { - pb.PrimitiveComposer.SetLikeAbleColumns(pb.PrimitiveComposer.GetProvider().GetLikeableColumns(nodeTypeUpperCase)) - } - colNames, err := parserutil.ExtractShowColNames(node.ShowTablesOpt) - if err != nil { - return err - } - colUsages, err := parserutil.ExtractShowColUsage(node.ShowTablesOpt) - if err != nil { - return err - } - switch nodeTypeUpperCase { + switch strings.ToUpper(node.Type) { case "AUTH": - // TODO - case "INSERT": - // SHOW has no comments or concept of await - err = pb.inferHeirarchyAndPersist(handlerCtx, node, nil, false, false) - if err != nil { - return err - } - case "METHODS": - // SHOW has no comments or concept of await - err = pb.inferHeirarchyAndPersist(handlerCtx, node, nil, false, false) - if err != nil { - return err - } - tbl, err := pb.PrimitiveComposer.GetTable(node) - if err != nil { - return err - } - currentService, err := tbl.GetServiceStr() - if err != nil { - return err - } - currentResource, err := tbl.GetResourceStr() - if err != nil { - return err - } - _, err = checkResource(handlerCtx, pb.PrimitiveComposer.GetProvider(), currentService, currentResource) - if err != nil { - return err - } - if node.ShowTablesOpt != nil { - meth := formulation.NewEmptyOperationStore() - err = pb.analyzeShowFilter(node, meth) - if err != nil { - return err - } - } - return nil - case "PROVIDERS": - // TODO + return pb.inferProviderForShow(node, handlerCtx) case "DEPENDENCIES": - if node.HasOnTable() || (node.ShowTablesOpt != nil && node.ShowTablesOpt.Filter != nil && - node.ShowTablesOpt.Filter.Filter != nil) { + if node.HasOnTable() || (node.ShowTablesOpt != nil && + node.ShowTablesOpt.Filter != nil && node.ShowTablesOpt.Filter.Filter != nil) { return fmt.Errorf("SHOW DEPENDENCIES supports only an optional LIKE clause") } - case "VERSION", "CONTRIBUTORS": - // no further analysis required - case "RESOURCES": - prov, err := handlerCtx.GetProvider(node.OnTable.Qualifier.GetRawVal()) - if err != nil { - return err - } - pb.PrimitiveComposer.SetProvider(prov) - _, err = pb.assembleResources(handlerCtx, pb.PrimitiveComposer.GetProvider(), node.OnTable.Name.GetRawVal()) - if err != nil { - return err - } - for _, col := range colNames { - if !formulation.ResourceKeyExists(col) { - return fmt.Errorf("SHOW key = '%s' does NOT exist", col) - } - } - for _, colUsage := range colUsages { - if !formulation.ResourceKeyExists(colUsage.ColName.Name.GetRawVal()) { - return fmt.Errorf("SHOW key = '%s' does NOT exist", colUsage.ColName.Name.GetRawVal()) - } - usageErr := parserutil.CheckSQLParserTypeVsResourceColumn(colUsage) - if usageErr != nil { - return usageErr - } - } - if node.ShowTablesOpt != nil { - rsc := formulation.NewEmptyResource() - err = pb.analyzeShowFilter(node, rsc) - if err != nil { - return err - } - } - case "SERVICES": - prov, err := handlerCtx.GetProvider(node.OnTable.Name.GetRawVal()) - if err != nil { - return err - } - pb.PrimitiveComposer.SetProvider(prov) - for _, col := range colNames { - if !formulation.ServiceKeyExists(col) { - return fmt.Errorf("SHOW key = '%s' does NOT exist", col) - } - } - for _, colUsage := range colUsages { - if !formulation.ServiceKeyExists(colUsage.ColName.Name.GetRawVal()) { - return fmt.Errorf("SHOW key = '%s' does NOT exist", colUsage.ColName.Name.GetRawVal()) - } - usageErr := parserutil.CheckSQLParserTypeVsServiceColumn(colUsage) - if usageErr != nil { - return usageErr - } - } - if node.ShowTablesOpt != nil { - svc := formulation.NewEmptyProviderService() - err = pb.analyzeShowFilter(node, svc) - if err != nil { - return err - } - } + return nil + case "VERSION", "CONTRIBUTORS", "PROVIDERS": + return nil default: - err = fmt.Errorf("SHOW statement not supported for '%s'", nodeTypeUpperCase) + return fmt.Errorf("SHOW %s must be planned through omnisdk", node.Type) } - return err +} + +func (pb *standardPrimitiveGenerator) analyzeDescribe(pbi planbuilderinput.PlanBuilderInput) error { + if _, handled := intrinsic.GeneratePrimitiveFunc(pbi.GetHandlerCtx(), pbi.GetStatement()); handled { + return nil + } + return fmt.Errorf("cannot resolve metadata for %s", sqlparser.String(pbi.GetStatement())) } diff --git a/internal/stackql/primitivegenerator/unary_selection.go b/internal/stackql/primitivegenerator/unary_selection.go deleted file mode 100644 index 2cdefbf8b..000000000 --- a/internal/stackql/primitivegenerator/unary_selection.go +++ /dev/null @@ -1,305 +0,0 @@ -package primitivegenerator - -import ( - "fmt" - "strings" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/astvisit" - "github.com/stackql/stackql/internal/stackql/docparser" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/planbuilderinput" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/util" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -//nolint:funlen // apathy -func (pb *standardPrimitiveGenerator) assembleUnarySelectionBuilder( - pbi planbuilderinput.PlanBuilderInput, - handlerCtx handler.HandlerContext, - node sqlparser.SQLNode, - rewrittenWhere *sqlparser.Where, - hIDs internaldto.HeirarchyIdentifiers, - schema formulation.Schema, - tbl tablemetadata.ExtendedTableMetadata, - selectTabulation formulation.Tabulation, - insertTabulation formulation.Tabulation, - cols []parserutil.ColumnHandle, - methodAnalysisOutput formulation.MethodAnalysisOutput, -) error { - inputTableName, err := tbl.GetInputTableName() - if err != nil { - return err - } - annotatedInsertTabulation := util.NewAnnotatedTabulation(insertTabulation, hIDs, inputTableName, "") - - prov, err := tbl.GetProviderObject() - if err != nil { - return err - } - svc, err := tbl.GetService() - if err != nil { - return err - } - resource, err := tbl.GetResource() - if err != nil { - return err - } - method, err := tbl.GetMethod() - if err != nil { - return err - } - _, err = docparser.OpenapiStackQLTabulationsPersistor( - prov, - svc, - resource, - method, - methodAnalysisOutput.IsAwait(), - []util.AnnotatedTabulation{annotatedInsertTabulation}, - pb.PrimitiveComposer.GetSQLEngine(), - prov.GetName(), - handlerCtx.GetNamespaceCollection(), - handlerCtx.GetControlAttributes(), - handlerCtx.GetSQLSystem(), - handlerCtx.GetPersistenceSystem(), - handlerCtx.GetTypingConfig(), - ) - if err != nil && !methodAnalysisOutput.IsNilResponseAllowed() { - return err - } - ctrs := pbi.GetTxnCtrlCtrs() - insPsc, err := pb.PrimitiveComposer.GetDRMConfig().GenerateInsertDML( - annotatedInsertTabulation, - prov, - svc, - resource, - method, - ctrs, - methodAnalysisOutput.IsNilResponseAllowed(), - methodAnalysisOutput.IsAwait(), - ) - if err != nil { - return err - } - pb.PrimitiveComposer.SetTxnCtrlCtrs(insPsc.GetGCCtrlCtrs()) - for _, col := range cols { - foundSchema := schema.FindByPath(col.Name, nil) - cc, ok := method.GetParameter(col.Name) - if foundSchema == nil && col.IsColumn { - if !(ok && cc.GetName() == col.Name) { - return fmt.Errorf( - "column = '%s' is NOT present in either: - data returned from provider, - acceptable parameters, use the DESCRIBE command to view available fields for SELECT operations", //nolint:lll // long string - col.Name) - } - } - if foundSchema != nil { - // A wire-spelled projection selects the display column (any-sdk #131). - col = col.WithDisplayName(insertTabulation) - } - selectTabulation.PushBackColumn( - formulation.NewColumnDescriptor( - col.Alias, - col.Name, - col.Qualifier, - col.DecoratedColumn, - col.Expr, - foundSchema, - col.Val, - ), - ) - } - selectSuffix := astvisit.GenerateModifiedSelectSuffix( - pbi.GetAnnotatedAST(), - node, - handlerCtx.GetSQLSystem(), - handlerCtx.GetASTFormatter(), - handlerCtx.GetNamespaceCollection(), - ) - selPsc, err := pb.PrimitiveComposer.GetDRMConfig().GenerateSelectDML( - util.NewAnnotatedTabulation(selectTabulation, hIDs, inputTableName, tbl.GetAlias()), - prov, - svc, - resource, - method, - insPsc.GetGCCtrlCtrs(), - selectSuffix, - astvisit.GenerateModifiedWhereClause( - pbi.GetAnnotatedAST(), - rewrittenWhere, - handlerCtx.GetSQLSystem(), - handlerCtx.GetASTFormatter(), - handlerCtx.GetNamespaceCollection(), - ), - methodAnalysisOutput.IsAwait(), - ) - if err != nil { - return err - } - pb.PrimitiveComposer.SetInsertPreparedStatementCtx(insPsc) - pb.PrimitiveComposer.SetSelectPreparedStatementCtx(selPsc) - pb.PrimitiveComposer.SetColumnOrder(cols) - return nil -} - -func (pb *standardPrimitiveGenerator) analyzeUnarySelection( - pbi planbuilderinput.PlanBuilderInput, - handlerCtx handler.HandlerContext, - node sqlparser.SQLNode, - rewrittenWhere *sqlparser.Where, - tbl tablemetadata.ExtendedTableMetadata, - cols []parserutil.ColumnHandle, - methodAnalysisOutput formulation.MethodAnalysisOutput, -) error { - _, err := tbl.GetProvider() - if err != nil { - return err - } - method, err := tbl.GetMethod() - if err != nil { - return err - } - schema, mediaType, err := tbl.GetResponseSchemaAndMediaType() - if err != nil { - return err - } - itemObjS, selectItemsKey, err := schema.GetSelectSchema(tbl.LookupSelectItemsKey(), mediaType) - // rscStr, _ := tbl.GetResourceStr() - unsuitableSchemaMsg := "analyzeUnarySelection(): schema unsuitable for select query" - if err != nil { - return fmt.Errorf("%s", unsuitableSchemaMsg) - } - tbl.SetSelectItemsKey(selectItemsKey) - provStr, _ := tbl.GetProviderStr() - svcStr, _ := tbl.GetServiceStr() - // rscStr, _ := tbl.GetResourceStr() - if itemObjS == nil { - return fmt.Errorf("%s", unsuitableSchemaMsg) - } - resource, err := tbl.GetResource() - if err != nil { - return err - } - prov, err := tbl.GetProviderObject() - if err != nil { - return err - } - svc, err := tbl.GetService() - if err != nil { - return err - } - if len(cols) == 0 { - addressSpaceFormulator := formulation.NewAddressSpaceFormulator( - formulation.NewAddressSpaceGrammar(), - prov, - svc, - resource, - method, - method.GetProjections(), - methodAnalysisOutput.IsAwait(), - ) - addressSpaceErr := addressSpaceFormulator.Formulate() - if addressSpaceErr != nil { - return addressSpaceErr - } - addressSpace := addressSpaceFormulator.GetAddressSpace() - if addressSpace == nil { - return fmt.Errorf("failed to obtain address space") - } - inferredRelation, inferredRelationErr := addressSpace.ToRelation( - formulation.NewStandardAddressSpaceExpansionConfig( - methodAnalysisOutput.IsAwait(), - true, // TODO: switch this off at the appropriate time - false, - )) - if inferredRelationErr != nil { - return inferredRelationErr - } - colz := inferredRelation.GetColumns() - for _, v := range colz { - cols = append(cols, parserutil.NewUnaliasedColumnHandle(v.GetName())) - } - } - insertTabulation := itemObjS.Tabulate(false, "") - - hIDs := internaldto.NewHeirarchyIdentifiers(provStr, svcStr, itemObjS.GetName(), "") - viewDTO, isView := handlerCtx.GetSQLSystem().GetViewByName(hIDs.GetTableName()) - if isView { - hIDs = hIDs.WithView(viewDTO) - } - selectTabulation := itemObjS.Tabulate(true, "") - - return pb.assembleUnarySelectionBuilder( - pbi, - handlerCtx, - node, - rewrittenWhere, - hIDs, - schema, - tbl, - selectTabulation, - insertTabulation, - cols, - methodAnalysisOutput, - ) -} - -func (pb *standardPrimitiveGenerator) analyzeUnaryAction( - pbi planbuilderinput.PlanBuilderInput, - handlerCtx handler.HandlerContext, - node sqlparser.SQLNode, - rewrittenWhere *sqlparser.Where, - tbl tablemetadata.ExtendedTableMetadata, - cols []parserutil.ColumnHandle, - methodAnalysisOutput formulation.MethodAnalysisOutput, -) error { - insertTabulation := methodAnalysisOutput.GetInsertTabulation() - selectTabulation := methodAnalysisOutput.GetSelectTabulation() - // method := methodAnalysisOutput.GetMethod() - // schema := methodAnalysisOutput.GetSchema() - - // inputTableName, err := tbl.GetInputTableName() - // if err != nil { - // return err - // } - rawhIDs := tbl.GetHeirarchyObjects().GetHeirarchyIDs() - itemObjS, _ := methodAnalysisOutput.GetItemSchema() - // TODO: handle nil response - itemSchemaName := "" - if itemObjS != nil { - itemSchemaName = itemObjS.GetName() - } - publishedMethodKey := "" - switch node := node.(type) { - case *sqlparser.Insert: - publishedMethodKey = node.Action - case *sqlparser.Update: - publishedMethodKey = node.Action - case *sqlparser.Delete: - publishedMethodKey = "delete" - default: - } - hIDs := internaldto.NewHeirarchyIdentifiers( - rawhIDs.GetProviderStr(), rawhIDs.GetServiceStr(), itemSchemaName, strings.ToLower(publishedMethodKey)) - - schema, _, err := tbl.GetResponseSchemaAndMediaType() - if err != nil && !methodAnalysisOutput.IsNilResponseAllowed() { - return err - } - return pb.assembleUnarySelectionBuilder( - pbi, - handlerCtx, - node, - rewrittenWhere, - hIDs, - schema, - tbl, - selectTabulation, - insertTabulation, - cols, - methodAnalysisOutput, - ) -} diff --git a/internal/stackql/provider/auth.go b/internal/stackql/provider/auth.go new file mode 100644 index 000000000..6244843af --- /dev/null +++ b/internal/stackql/provider/auth.go @@ -0,0 +1,29 @@ +package provider + +import ( + "errors" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" +) + +func (gp *GenericProvider) CheckCredentialFile(authCtx *dto.AuthCtx) error { + return omnisdk.NewCredential(gp.GetProviderString(), *dto.OmniAuth(authCtx)).Check() +} + +func (gp *GenericProvider) ShowAuth(authCtx *dto.AuthCtx) (*omnisdk.AuthMetadata, error) { + if authCtx == nil { + return nil, errors.New(constants.NotAuthenticatedShowStr) + } + principal, err := omnisdk.NewCredential(gp.GetProviderString(), *dto.OmniAuth(authCtx)).Principal() + if err != nil { + return nil, err + } + return &omnisdk.AuthMetadata{ + Principal: principal, Type: strings.ToUpper(authCtx.Type), + Source: authCtx.GetCredentialsSourceDescriptorString(), + }, nil +} diff --git a/internal/stackql/provider/generic.go b/internal/stackql/provider/generic.go index 70eb4461c..315616c19 100644 --- a/internal/stackql/provider/generic.go +++ b/internal/stackql/provider/generic.go @@ -1,481 +1,33 @@ package provider import ( - "errors" "fmt" - "github.com/stackql/any-sdk/pkg/auth_util" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/netutils" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/docparser" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/methodselect" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/relational" - - "github.com/stackql/stackql/pkg/sqltypeutil" - - sdk_internal_dto "github.com/stackql/any-sdk/pkg/internaldto" - - "net/http" - "regexp" - "strings" -) - -var ( - //nolint:unused // prefer declarative - gitHubLinksNextRegex *regexp.Regexp = regexp.MustCompile(`.*<(?P[^>]*)>;\ rel="next".*`) + "github.com/stackql/stackql/internal/stackql/registry" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/sql_system" ) +// GenericProvider retains host registration metadata, never request execution. type GenericProvider struct { - provider formulation.Provider - runtimeCtx dto.RuntimeCtx - currentService string - discoveryAdapter formulation.IDiscoveryAdapter - apiVersion string - methodSelector methodselect.IMethodSelector - authUtil auth_util.AuthUtility - defaultClient *http.Client // for testing purposes only, defaulted downstream -} - -func (gp *GenericProvider) GetDefaultKeyForDeleteItems() string { - if gp.provider.GetDeleteItemsKey() != "" { - return gp.provider.GetDeleteItemsKey() - } - return "items" -} - -func (gp *GenericProvider) GetMethodSelector() methodselect.IMethodSelector { - return gp.methodSelector -} - -func (gp *GenericProvider) GetVersion() string { - return gp.apiVersion -} - -func (gp *GenericProvider) GetServiceShard( - serviceKey string, - resourceKey string, - runtimeCtx dto.RuntimeCtx, //nolint:revive // future proofing -) (formulation.Service, error) { - return gp.discoveryAdapter.GetServiceShard(gp.provider, serviceKey, resourceKey) -} - -//nolint:revive // future proofing -func (gp *GenericProvider) PersistStaticExternalSQLDataSource(runtimeCtx dto.RuntimeCtx) error { - return gp.discoveryAdapter.PersistStaticExternalSQLDataSource(gp.provider) -} - -func (gp *GenericProvider) inferAuthType(authCtx dto.AuthCtx, authTypeRequested string) string { - ft := strings.ToLower(authTypeRequested) - switch ft { - case dto.AuthAzureDefaultStr: - return dto.AuthAzureDefaultStr - case dto.AuthAPIKeyStr: - return dto.AuthAPIKeyStr - case dto.AuthBasicStr: - return dto.AuthBasicStr - case dto.AuthBearerStr: - return dto.AuthBearerStr - case dto.AuthServiceAccountStr: - return dto.AuthServiceAccountStr - case dto.AuthInteractiveStr: - return dto.AuthInteractiveStr - case dto.AuthNullStr: - return dto.AuthNullStr - case dto.AuthAWSSigningv4Str: - return dto.AuthAWSSigningv4Str - case dto.AuthCustomStr: - return dto.AuthCustomStr - case dto.OAuth2Str: - return dto.OAuth2Str - } - if authCtx.KeyFilePath != "" || authCtx.KeyEnvVar != "" { - return dto.AuthServiceAccountStr - } - return dto.AuthNullStr -} - -func (gp *GenericProvider) Auth( - authCtx *dto.AuthCtx, - authTypeRequested string, - enforceRevokeFirst bool, -) (*http.Client, error) { - authCtx = authCtx.Clone() - at := gp.inferAuthType(*authCtx, authTypeRequested) - switch at { - case dto.AuthAPIKeyStr: - return gp.authUtil.ApiTokenAuth(authCtx, gp.runtimeCtx, false) - case dto.AuthBearerStr: - return gp.authUtil.ApiTokenAuth(authCtx, gp.runtimeCtx, true) - case dto.AuthServiceAccountStr: - scopes := authCtx.Scopes - return gp.authUtil.GoogleOauthServiceAccount(gp.GetProviderString(), authCtx, scopes, gp.runtimeCtx) - case dto.OAuth2Str: - if authCtx.GrantType == dto.ClientCredentialsStr { - scopes := authCtx.Scopes - return gp.authUtil.GenericOauthClientCredentials(authCtx, scopes, gp.runtimeCtx) - } - case dto.AuthBasicStr: - return gp.authUtil.BasicAuth(authCtx, gp.runtimeCtx) - case dto.AuthCustomStr: - return gp.authUtil.CustomAuth(authCtx, gp.runtimeCtx) - case dto.AuthAzureDefaultStr: - return gp.authUtil.AzureDefaultAuth(authCtx, gp.runtimeCtx) - case dto.AuthInteractiveStr: - return gp.authUtil.GCloudOAuth(gp.runtimeCtx, authCtx, enforceRevokeFirst) - case dto.AuthAWSSigningv4Str: - return gp.authUtil.AwsSigningAuth(authCtx, gp.runtimeCtx) - case dto.AuthNullStr: - return netutils.GetHTTPClient(gp.runtimeCtx, http.DefaultClient), nil - } - return nil, fmt.Errorf("could not infer auth type") -} - -func (gp *GenericProvider) AuthRevoke(authCtx *dto.AuthCtx) error { - return gp.authUtil.AuthRevoke(authCtx) -} - -func (gp *GenericProvider) GetMethodForAction( - serviceName string, - resourceName string, - iqlAction string, - parameters parserutil.ColumnKeyedDatastore, - runtimeCtx dto.RuntimeCtx, -) (formulation.StandardOperationStore, string, error) { - rsc, err := gp.GetResource(serviceName, resourceName, runtimeCtx) - if err != nil { - return nil, "", err - } - return gp.methodSelector.GetMethodForAction(rsc, iqlAction, parameters) -} - -func (gp *GenericProvider) GetDefaultHTTPClient() *http.Client { - return gp.defaultClient -} - -func (gp *GenericProvider) GetFirstMethodForAction( - serviceName string, - resourceName string, - iqlAction string, - runtimeCtx dto.RuntimeCtx, -) (formulation.StandardOperationStore, string, error) { - rsc, err := gp.GetResource(serviceName, resourceName, runtimeCtx) - if err != nil { - return nil, "", err - } - rv, str, ok := rsc.GetFirstMethodFromSQLVerb(iqlAction) - if !ok { - return nil, "", fmt.Errorf("cannot locate method for action '%s'", iqlAction) - } - return rv, str, nil -} - -func (gp *GenericProvider) InferDescribeMethod( - rsc formulation.Resource, -) (formulation.StandardOperationStore, string, error) { - if rsc == nil { - return nil, "", fmt.Errorf("cannot infer describe method from nil resource") - } - m, mk, ok := rsc.GetFirstMethodFromSQLVerb("select") - if ok { - return m, mk, nil - } - return nil, "", fmt.Errorf( - //nolint:lll // long message - "SELECT not supported for this resource, use SHOW METHODS to view available operations for the resource and then invoke a supported method using the EXEC command") -} - -func (gp *GenericProvider) GetObjectSchema( - serviceName string, - resourceName string, - schemaName string, -) (formulation.Schema, error) { - svc, err := gp.GetServiceShard(serviceName, resourceName, gp.runtimeCtx) - if err != nil { - return nil, err - } - return svc.GetSchema(schemaName) -} - -func (gp *GenericProvider) ShowAuth(authCtx *dto.AuthCtx) (*formulation.AuthMetadata, error) { - var err error - var retVal formulation.AuthMetadata - var authObj formulation.AuthMetadata - if authCtx == nil { - return nil, errors.New(constants.NotAuthenticatedShowStr) //nolint:stylecheck // happy with message - } - switch gp.inferAuthType(*authCtx, authCtx.Type) { - case dto.AuthServiceAccountStr: - sa, saErr := gp.authUtil.ParseServiceAccountFile(authCtx) - if saErr == nil { - authObj = formulation.AuthMetadata{ - Principal: sa.Email, - Type: strings.ToUpper(dto.AuthServiceAccountStr), - Source: authCtx.GetCredentialsSourceDescriptorString(), - } - retVal = authObj - gp.authUtil.ActivateAuth(authCtx, sa.Email, dto.AuthServiceAccountStr) - } - case dto.AuthInteractiveStr: - principal, sdkErr := gp.authUtil.GetCurrentGCloudOauthUser() - if sdkErr == nil { - principalStr := string(principal) - if principalStr != "" { - authObj = formulation.AuthMetadata{ - Principal: principalStr, - Type: strings.ToUpper(dto.AuthInteractiveStr), - Source: "OAuth", - } - retVal = authObj - gp.authUtil.ActivateAuth(authCtx, principalStr, dto.AuthInteractiveStr) - } else { - err = errors.New(constants.NotAuthenticatedShowStr) //nolint:stylecheck // happy with message - } - } else { - logging.GetLogger().Infoln(sdkErr) - err = errors.New(constants.NotAuthenticatedShowStr) //nolint:stylecheck // happy with message - } - default: - err = errors.New(constants.NotAuthenticatedShowStr) //nolint:stylecheck // happy with message - } - return &retVal, err -} - -func (gp *GenericProvider) GetLikeableColumns(tableName string) []string { - var retVal []string - switch tableName { - case "SERVICES": - return []string{ - "id", - "name", - } - case "RESOURCES": - return []string{ - "id", - "name", - } - case "METHODS": - return []string{ - "id", - "name", - } - case "PROVIDERS": - return []string{ - "name", - } - } - return retVal -} - -func (gp *GenericProvider) EnhanceMetadataFilter( - metadataType string, - metadataFilter func(formulation.ITable) (formulation.ITable, error), - colsVisited map[string]bool, -) (func(formulation.ITable) (formulation.ITable, error), error) { - typeVisited, typeOk := colsVisited["type"] - preferredVisited, preferredOk := colsVisited["preferred"] - sqlTrue, sqlTrueErr := sqltypeutil.InterfaceToSQLType(true) - sqlCloudStr, sqlCloudStrErr := sqltypeutil.InterfaceToSQLType("cloud") - equalsOperator, operatorErr := relational.GetOperatorPredicate("=") - if sqlTrueErr != nil || sqlCloudStrErr != nil || operatorErr != nil { - return nil, - //nolint:revive,stylecheck // exclamation marks for egregious error - fmt.Errorf("typing and operator system broken!!!") - } - switch metadataType { //nolint:gocritic // happy to have this as a switch - case "service": - if typeOk && typeVisited && preferredOk && preferredVisited { - return metadataFilter, nil - } - if typeOk && typeVisited { - return relational.AndTableFilters( - metadataFilter, - relational.ConstructTablePredicateFilter("preferred", sqlTrue, equalsOperator), - ), nil - } - if preferredOk && preferredVisited { - return relational.AndTableFilters( - metadataFilter, - relational.ConstructTablePredicateFilter("type", sqlCloudStr, equalsOperator), - ), nil - } - return relational.AndTableFilters( - relational.AndTableFilters( - metadataFilter, - relational.ConstructTablePredicateFilter("cloud", sqlCloudStr, equalsOperator), - ), - relational.ConstructTablePredicateFilter("preferred", sqlTrue, equalsOperator), - ), nil - } - return metadataFilter, nil + document registry.ProviderDocument + sqlSystem sql_system.SQLSystem } -func (gp *GenericProvider) getProviderServices() (map[string]formulation.ProviderService, error) { - retVal := make(map[string]formulation.ProviderService) - disDoc, err := gp.discoveryAdapter.GetServiceHandlesMap(gp.provider) +func (p *GenericProvider) GetProviderString() string { return p.document.GetName() } +func (p *GenericProvider) GetProvider() (registry.ProviderDocument, error) { return p.document, nil } +func (p *GenericProvider) PersistStaticExternalSQLDataSource(_ dto.RuntimeCtx) error { + tables, err := p.document.ExternalTables() if err != nil { - return nil, err - } - for k, item := range disDoc { - retVal[docparser.TranslateServiceKeyGenericProviderToIql(k)] = item - } - return retVal, nil -} - -//nolint:revive // future proofing -func (gp *GenericProvider) GetProviderServicesRedacted( - runtimeCtx dto.RuntimeCtx, - extended bool, -) (map[string]formulation.ProviderService, error) { - return gp.getProviderServices() -} - -//nolint:revive // future proofing -func (gp *GenericProvider) GetResourcesRedacted( - currentService string, - runtimeCtx dto.RuntimeCtx, - extended bool, -) (map[string]formulation.Resource, error) { - svcDiscDocMap, err := gp.discoveryAdapter.GetResourcesMap(gp.provider, currentService) - return svcDiscDocMap, err -} - -func (gp *GenericProvider) CheckCredentialFile(authCtx *dto.AuthCtx) error { - switch authCtx.Type { - case dto.AuthServiceAccountStr: - _, err := gp.authUtil.ParseServiceAccountFile(authCtx) - return err - case dto.AuthAPIKeyStr: - _, err := authCtx.GetCredentialsBytes() return err } - return fmt.Errorf("auth type = '%s' not supported", authCtx.Type) -} - -func (gp *GenericProvider) SetCurrentService(serviceKey string) { - gp.currentService = serviceKey -} - -func (gp *GenericProvider) GetCurrentService() string { - return gp.currentService -} - -//nolint:revive // future proofing -func (gp *GenericProvider) GetResourcesMap( - serviceKey string, - runtimeCtx dto.RuntimeCtx, -) (map[string]formulation.Resource, error) { - return gp.discoveryAdapter.GetResourcesMap(gp.provider, serviceKey) -} - -func (gp *GenericProvider) GetResource( - serviceKey string, - resourceKey string, - runtimeCtx dto.RuntimeCtx, -) (formulation.Resource, error) { - svc, err := gp.GetServiceShard(serviceKey, resourceKey, runtimeCtx) - if err != nil { - return nil, err - } - return svc.GetResource(resourceKey) -} - -func (gp *GenericProvider) GetProviderString() string { - return gp.provider.GetName() -} - -func (gp *GenericProvider) GetProvider() (formulation.Provider, error) { - if gp.provider == nil { - return nil, fmt.Errorf("nil provider object") - } - return gp.provider, nil -} - -func (gp *GenericProvider) InferMaxResultsElement(formulation.OperationStore) sdk_internal_dto.HTTPElement { - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.QueryParam, - "maxResults", - ) -} - -func (gp *GenericProvider) InferNextPageRequestElement(ho internaldto.Heirarchy) sdk_internal_dto.HTTPElement { - st, ok := gp.getPaginationRequestTokenSemantic(ho) - if ok { - if tp, err := sdk_internal_dto.ExtractHTTPElement(st.GetLocation()); err == nil { - rv := sdk_internal_dto.NewHTTPElement( - tp, - st.GetKey(), - ) - transformer, tErr := st.GetTransformer() - if tErr == nil && transformer != nil { - rv.SetTransformer(transformer) - } - return rv - } - } - switch gp.GetProviderString() { - case "github", "okta": - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.RequestString, - "", - ) - default: - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.QueryParam, - "pageToken", - ) - } -} - -func (gp *GenericProvider) getPaginationRequestTokenSemantic( - ho internaldto.Heirarchy, -) (formulation.TokenSemantic, bool) { - if ho.GetMethod() == nil { - return nil, false + if len(tables) == 0 { + return fmt.Errorf("no external tables supplied") } - return ho.GetMethod().GetPaginationRequestTokenSemantic() -} - -func (gp *GenericProvider) getPaginationResponseTokenSemantic( - ho internaldto.Heirarchy, -) (formulation.TokenSemantic, bool) { - if ho.GetMethod() == nil { - return nil, false - } - return ho.GetMethod().GetPaginationResponseTokenSemantic() -} - -func (gp *GenericProvider) InferNextPageResponseElement(ho internaldto.Heirarchy) sdk_internal_dto.HTTPElement { - st, ok := gp.getPaginationResponseTokenSemantic(ho) - if ok { - if tp, err := sdk_internal_dto.ExtractHTTPElement(st.GetLocation()); err == nil { - rv := sdk_internal_dto.NewHTTPElement( - tp, - st.GetKey(), - ) - transformer, tErr := st.GetTransformer() - if tErr == nil && transformer != nil { - rv.SetTransformer(transformer) - } - return rv + for _, table := range tables { + if err = p.sqlSystem.RegisterExternalTable(p.GetProviderString(), table); err != nil { + return err } } - switch gp.GetProviderString() { - case "github", "okta": - rv := sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.Header, - "Link", - ) - rv.SetTransformer(formulation.DefaultLinkHeaderTransformer) - return rv - default: - return sdk_internal_dto.NewHTTPElement( - sdk_internal_dto.BodyAttribute, - "nextPageToken", - ) - } + return nil } diff --git a/internal/stackql/provider/provider.go b/internal/stackql/provider/provider.go index 98fce4210..e6f27626d 100644 --- a/internal/stackql/provider/provider.go +++ b/internal/stackql/provider/provider.go @@ -1,184 +1,29 @@ package provider import ( - "net/http" - - "github.com/stackql/any-sdk/pkg/auth_util" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/docparser" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/methodselect" - "github.com/stackql/stackql/internal/stackql/parserutil" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/registry" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" - - sdk_internal_dto "github.com/stackql/any-sdk/pkg/internaldto" -) - -const ( - ambiguousServiceErrorMessage string = "More than one service exists with this name, please use the id in the object name, or unset the --usenonpreferredapis flag" //nolint:lll // long string - SchemaDelimiter string = docparser.SchemaDelimiter ) -var ( - DummyAuth bool = false //nolint:revive,gochecknoglobals // prefer declarative -) - -type ProviderParam struct { //nolint:revive // TODO: review - Id string //nolint:revive,stylecheck // TODO: review - Type string - Format string -} +// DummyAuth is retained for the existing test harness; execution authentication is owned by Omni. +var DummyAuth bool //nolint:gochecknoglobals // legacy test harness type IProvider interface { - Auth(authCtx *dto.AuthCtx, authTypeRequested string, enforceRevokeFirst bool) (*http.Client, error) - - AuthRevoke(authCtx *dto.AuthCtx) error - - CheckCredentialFile(authCtx *dto.AuthCtx) error - - GetDefaultHTTPClient() *http.Client - - EnhanceMetadataFilter( - string, - func(formulation.ITable) (formulation.ITable, error), - map[string]bool) (func(formulation.ITable) (formulation.ITable, error), error) - - GetCurrentService() string - - GetDefaultKeyForDeleteItems() string - - GetFirstMethodForAction( - serviceName string, - resourceName string, - iqlAction string, - runtimeCtx dto.RuntimeCtx) (formulation.StandardOperationStore, string, error) - - GetLikeableColumns(string) []string - - GetMethodForAction( - serviceName string, - resourceName string, - iqlAction string, - parameters parserutil.ColumnKeyedDatastore, - runtimeCtx dto.RuntimeCtx) (formulation.StandardOperationStore, string, error) - - GetMethodSelector() methodselect.IMethodSelector - - GetProvider() (formulation.Provider, error) - + CheckCredentialFile(*dto.AuthCtx) error + ShowAuth(*dto.AuthCtx) (*omnisdk.AuthMetadata, error) GetProviderString() string - - GetProviderServicesRedacted( - runtimeCtx dto.RuntimeCtx, - extended bool) (map[string]formulation.ProviderService, error) - - GetResource(serviceKey string, resourceKey string, runtimeCtx dto.RuntimeCtx) (formulation.Resource, error) - - GetResourcesMap( - serviceKey string, - runtimeCtx dto.RuntimeCtx) (map[string]formulation.Resource, error) - - GetResourcesRedacted( - currentService string, - runtimeCtx dto.RuntimeCtx, - extended bool) (map[string]formulation.Resource, error) - - GetServiceShard(serviceKey string, resourceKey string, runtimeCtx dto.RuntimeCtx) (formulation.Service, error) - - GetObjectSchema(serviceName string, resourceName string, schemaName string) (formulation.Schema, error) - - GetVersion() string - - InferDescribeMethod(formulation.Resource) (formulation.StandardOperationStore, string, error) - - InferMaxResultsElement(formulation.OperationStore) sdk_internal_dto.HTTPElement - - InferNextPageRequestElement(internaldto.Heirarchy) sdk_internal_dto.HTTPElement - - InferNextPageResponseElement(internaldto.Heirarchy) sdk_internal_dto.HTTPElement - + GetProvider() (registry.ProviderDocument, error) PersistStaticExternalSQLDataSource(dto.RuntimeCtx) error - - SetCurrentService(serviceKey string) - - ShowAuth(authCtx *dto.AuthCtx) (*formulation.AuthMetadata, error) } -//nolint:revive // TODO: review func GenerateProvider( - runtimeCtx dto.RuntimeCtx, - providerStr, - providerVersion string, - reg formulation.RegistryAPI, - sqlSystem sql_system.SQLSystem, - persistenceSystem formulation.PersistenceSystem, - defaultHTTPClient *http.Client, -) (IProvider, error) { - switch providerStr { //nolint:gocritic // TODO: review - default: - return newGenericProvider( - runtimeCtx, providerStr, providerVersion, reg, sqlSystem, persistenceSystem, defaultHTTPClient) - } -} - -func getURL(prov string) (string, error) { //nolint:unparam // TODO: review - switch prov { - case "google": - return constants.GoogleV1DiscoveryDoc, nil - default: - return prov, nil - } -} - -//nolint:revive // TODO: review -func newGenericProvider( - rtCtx dto.RuntimeCtx, - providerStr, - versionStr string, - reg formulation.RegistryAPI, - _ sql_system.SQLSystem, - persistenceSystem formulation.PersistenceSystem, - defaultHTTPClient *http.Client, + _ dto.RuntimeCtx, name, version string, reg registry.RegistryAPI, system sql_system.SQLSystem, ) (IProvider, error) { - methSel, err := methodselect.NewMethodSelector(providerStr, versionStr) + doc, err := reg.LoadProviderByName(name, version) if err != nil { return nil, err } - - rootURL, err := getURL(providerStr) - if err != nil { - return nil, err - } - - da := formulation.NewBasicDiscoveryAdapter( - providerStr, - rootURL, - formulation.NewTTLDiscoveryStore( - persistenceSystem, - reg, - rtCtx, - ), - &rtCtx, - reg, - persistenceSystem, - ) - - p, err := da.GetProvider(rtCtx.ProviderStr) - - if err != nil { - return nil, err - } - - gp := &GenericProvider{ - provider: p, - runtimeCtx: rtCtx, - discoveryAdapter: da, - apiVersion: versionStr, - methodSelector: methSel, - authUtil: auth_util.NewAuthUtility(defaultHTTPClient), - defaultClient: defaultHTTPClient, - } - return gp, err + return &GenericProvider{document: doc, sqlSystem: system}, nil } diff --git a/internal/stackql/psqlwire/server.go b/internal/stackql/psqlwire/server.go index a97046d6e..209d2deb3 100644 --- a/internal/stackql/psqlwire/server.go +++ b/internal/stackql/psqlwire/server.go @@ -8,8 +8,8 @@ import ( "fmt" "github.com/sirupsen/logrus" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "gopkg.in/yaml.v2" "github.com/stackql/psql-wire/pkg/sqlbackend" diff --git a/internal/stackql/pushdown/pushdown.go b/internal/stackql/pushdown/pushdown.go deleted file mode 100644 index 257a8e030..000000000 --- a/internal/stackql/pushdown/pushdown.go +++ /dev/null @@ -1,262 +0,0 @@ -// Package pushdown computes, during the analysis/planning phase, the neutral query-option -// intent (projection / predicates / order-by / limit / offset / count) that an acquire can -// push to the upstream API. It is protocol-agnostic: stackql extracts the intent from the -// SELECT and any-sdk owns the dialect-specific translation and request application (via -// HTTPPreparator.WithPushdownIntent). Push-down is purely an optimisation - stackql's -// client-side WHERE / projection / LIMIT remain authoritative, so a partial or absent -// translation can never change results - and a no-op unless the method carries a -// queryParamPushdown config. -package pushdown - -import ( - "strconv" - "strings" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -// ComputeIntent extracts the neutral PushdownIntent from a SELECT for the supplied method. -// It returns (nil, false) when the method carries no queryParamPushdown config, the node is -// not a simple resource-scoped scan, or nothing translatable is present. The caller hands -// the intent to the HTTP preparator, which performs the dialect translation and applies the -// resulting query params to the request inside any-sdk. -func ComputeIntent(node sqlparser.SQLNode, op formulation.OperationStore) (formulation.PushdownIntent, bool) { - if op == nil { - return nil, false - } - if _, ok := op.GetQueryParamPushdown(); !ok { - return nil, false - } - return buildPushdownIntent(node) -} - -// buildPushdownIntent extracts the neutral PushdownIntent from a SELECT statement. The -// second return reports whether anything pushable was found. It is the pure AST-to-intent -// translation, independent of any method config (that gate lives in ComputeIntent). -func buildPushdownIntent(node sqlparser.SQLNode) (formulation.PushdownIntent, bool) { - sel, ok := node.(*sqlparser.Select) - if !ok || !isSimpleResourceScopedSelect(sel) { - return nil, false - } - - var predicates []formulation.PushdownPredicate - if sel.Where != nil { - predicates = collectPushdownPredicates(sel.Where.Expr) - } - - projection, count := extractProjectionAndCount(sel.SelectExprs) - if count { - // COUNT(*) collapses grain: push the WHERE (applied before counting) and the count - // itself, but never limit/offset/projection/order-by - they would change or misreport - // the count. SQL LIMIT here reverts to a client-side primitive. - return formulation.NewPushdownIntent(nil, predicates, nil, 0, false, 0, false, true), true - } - - var orderBy []formulation.PushdownOrder - for _, o := range sel.OrderBy { - if cn, isCol := o.Expr.(*sqlparser.ColName); isCol { - orderBy = append(orderBy, formulation.NewPushdownOrder( - cn.Name.GetRawVal(), - strings.Contains(strings.ToLower(o.Direction), "desc"), - )) - } - } - - // A pushed projection must also fetch every column the client-side WHERE / - // ORDER BY consult, else the authoritative re-filter drops all rows - // (issue #682). Output shaping is unchanged. - projection = unionReferencedColumns(projection, sel) - - limit, limitSet := 0, false - offset, offsetSet := 0, false - if sel.Limit != nil { - if v, limOk := sqlValAsInt(sel.Limit.Rowcount); limOk { - limit, limitSet = v, true - } - if v, offOk := sqlValAsInt(sel.Limit.Offset); offOk { - offset, offsetSet = v, true - } - } - - hasContent := len(projection) > 0 || len(predicates) > 0 || len(orderBy) > 0 || limitSet || offsetSet - if !hasContent { - return nil, false - } - return formulation.NewPushdownIntent( - projection, predicates, orderBy, limit, limitSet, offset, offsetSet, false), true -} - -// SelectLimit returns the integer LIMIT of a SELECT when it can be pushed to the upstream -// fetch: only for a simple, resource-scoped scan (one table, no join / GROUP BY / DISTINCT -// / HAVING). For a grain-changing or multi-set query the LIMIT stays a client-side primitive. -// It is used by the GraphQL acquire path to bound the page size. -func SelectLimit(node sqlparser.SQLNode) (int, bool) { - sel, ok := node.(*sqlparser.Select) - if !ok || sel.Limit == nil || !isSimpleResourceScopedSelect(sel) { - return 0, false - } - return sqlValAsInt(sel.Limit.Rowcount) -} - -// unionReferencedColumns extends a non-empty projection with WHERE / ORDER BY -// columns not already projected (issue #682); an empty projection is unchanged. -func unionReferencedColumns(projection []string, sel *sqlparser.Select) []string { - if len(projection) == 0 { - return projection - } - seen := make(map[string]struct{}, len(projection)) - for _, c := range projection { - seen[c] = struct{}{} - } - var walkTargets []sqlparser.SQLNode - if sel.Where != nil && sel.Where.Expr != nil { - walkTargets = append(walkTargets, sel.Where.Expr) - } - if sel.OrderBy != nil { - walkTargets = append(walkTargets, sel.OrderBy) - } - //nolint:errcheck // the visitor never errs - sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { - if cn, isCol := node.(*sqlparser.ColName); isCol { - name := cn.Name.GetRawVal() - if _, present := seen[name]; !present { - seen[name] = struct{}{} - projection = append(projection, name) - } - } - return true, nil - }, walkTargets...) - return projection -} - -// isSimpleResourceScopedSelect reports whether a SELECT is a single-resource scan with -// no grain change, i.e. safe to translate LIMIT/OFFSET/projection into server predicates. -func isSimpleResourceScopedSelect(sel *sqlparser.Select) bool { - if len(sel.From) != 1 { - return false - } - if _, isSimple := sel.From[0].(*sqlparser.AliasedTableExpr); !isSimple { - return false - } - return !sel.Distinct && len(sel.GroupBy) == 0 && sel.Having == nil -} - -// extractProjectionAndCount reads the SELECT list, returning the projected column -// names (nil for SELECT *) and whether a COUNT(*) aggregate is present. -func extractProjectionAndCount(exprs sqlparser.SelectExprs) ([]string, bool) { - var projection []string - var count bool - for _, e := range exprs { - ae, ok := e.(*sqlparser.AliasedExpr) - if !ok { - continue - } - switch inner := ae.Expr.(type) { - case *sqlparser.ColName: - projection = append(projection, inner.Name.GetRawVal()) - case *sqlparser.FuncExpr: - if strings.EqualFold(inner.Name.GetRawVal(), "count") { - count = true - } - } - } - return projection, count -} - -// collectPushdownPredicates flattens AND-conjoined simple comparisons into neutral -// predicates. Anything not translatable (OR, non-column LHS, unsupported operator) -// is simply omitted - it stays a client-side filter via the unchanged WHERE clause. -func collectPushdownPredicates(expr sqlparser.Expr) []formulation.PushdownPredicate { - switch e := expr.(type) { - case *sqlparser.AndExpr: - return append(collectPushdownPredicates(e.Left), collectPushdownPredicates(e.Right)...) - case *sqlparser.ComparisonExpr: - if p, ok := comparisonToPredicate(e); ok { - return []formulation.PushdownPredicate{p} - } - } - return nil -} - -func comparisonToPredicate(expr *sqlparser.ComparisonExpr) (formulation.PushdownPredicate, bool) { - col, ok := expr.Left.(*sqlparser.ColName) - if !ok { - return nil, false - } - val, ok := expr.Right.(*sqlparser.SQLVal) - if !ok { - return nil, false - } - column := col.Name.GetRawVal() - rawVal, valOk := sqlValAsComparable(val) - if !valOk { - return nil, false - } - switch expr.Operator { - case sqlparser.EqualStr, sqlparser.NotEqualStr, sqlparser.GreaterThanStr, - sqlparser.GreaterEqualStr, sqlparser.LessThanStr, sqlparser.LessEqualStr: - return formulation.NewPushdownPredicate(column, expr.Operator, rawVal), true - case sqlparser.LikeStr: - // Only a prefix pattern ('A%' with no other wildcard) maps to a startswith predicate. - s, isStr := rawVal.(string) - if isStr && strings.HasSuffix(s, "%") { - prefix := strings.TrimSuffix(s, "%") - if !strings.ContainsAny(prefix, "%_") { - return formulation.NewPushdownPredicate(column, "startswith", prefix), true - } - } - } - return nil, false -} - -// sqlValAsComparable renders a SQLVal as a Go value ApplyPushdown can format: -// integers as int, everything else as the raw string (quoted by any-sdk). -func sqlValAsComparable(v *sqlparser.SQLVal) (interface{}, bool) { - if v == nil { - return nil, false - } - switch v.Type { //nolint:exhaustive // only int/string are translated; others use the raw string - case sqlparser.IntVal: - if i, err := strconv.Atoi(string(v.Val)); err == nil { - return i, true - } - return string(v.Val), true - case sqlparser.StrVal: - return string(v.Val), true - default: - return string(v.Val), true - } -} - -func sqlValAsInt(expr sqlparser.Expr) (int, bool) { - v, ok := expr.(*sqlparser.SQLVal) - if !ok || v.Type != sqlparser.IntVal { - return 0, false - } - i, err := strconv.Atoi(string(v.Val)) - if err != nil { - return 0, false - } - return i, true -} - -// RedactHeaderValue masks a header value for logging: alphanumerics become 'x' and the -// result is truncated to at most 5 characters. Keys present in allowedInnocuous -// (lower case) pass through unredacted. -func RedactHeaderValue(key string, val string, allowedInnocuous map[string]bool) string { - if allowedInnocuous != nil && allowedInnocuous[strings.ToLower(key)] { - return val - } - const maxRedactedLen = 5 - runes := []rune(val) - if len(runes) > maxRedactedLen { - runes = runes[:maxRedactedLen] - } - for i, r := range runes { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { - runes[i] = 'x' - } - } - return string(runes) -} diff --git a/internal/stackql/pushdown/pushdown_test.go b/internal/stackql/pushdown/pushdown_test.go deleted file mode 100644 index 02c111b37..000000000 --- a/internal/stackql/pushdown/pushdown_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package pushdown //nolint:testpackage // tests unexported buildPushdownIntent - -import ( - "testing" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -func mustSelect(t *testing.T, sql string) sqlparser.SQLNode { - t.Helper() - stmt, err := sqlparser.Parse(sql) - if err != nil { - t.Fatalf("parse %q: %v", sql, err) - } - return stmt -} - -func TestBuildPushdownIntent_FullSelect(t *testing.T) { - node := mustSelect(t, - "select a, b from t where x = 'v' and y > 5 order by z desc limit 10 offset 2") - intent, ok := buildPushdownIntent(node) - if !ok { - t.Fatalf("expected pushable intent") - } - // The projection is the union of SELECT-list columns and the WHERE / ORDER BY - // referenced columns, in that order (issue #682). - proj := intent.GetProjection() - want := []string{"a", "b", "x", "y", "z"} - if len(proj) != len(want) { - t.Fatalf("projection = %v, want %v", proj, want) - } - for i := range want { - if proj[i] != want[i] { - t.Errorf("projection[%d] = %s, want %s", i, proj[i], want[i]) - } - } - preds := intent.GetPredicates() - if len(preds) != 2 { - t.Fatalf("predicates = %v", preds) - } - if preds[0].GetColumn() != "x" || preds[0].GetOperator() != "=" || preds[0].GetValue() != "v" { - t.Errorf("predicate[0] = col=%s op=%s val=%v", - preds[0].GetColumn(), preds[0].GetOperator(), preds[0].GetValue()) - } - if preds[1].GetColumn() != "y" || preds[1].GetOperator() != ">" || preds[1].GetValue() != 5 { - t.Errorf("predicate[1] = col=%s op=%s val=%v", - preds[1].GetColumn(), preds[1].GetOperator(), preds[1].GetValue()) - } - ob := intent.GetOrderBy() - if len(ob) != 1 || ob[0].GetColumn() != "z" || !ob[0].IsDescending() { - t.Errorf("orderBy = %v", ob) - } - if l, set := intent.GetLimit(); !set || l != 10 { - t.Errorf("limit = %d set=%v", l, set) - } - if o, set := intent.GetOffset(); !set || o != 2 { - t.Errorf("offset = %d set=%v", o, set) - } -} - -func TestBuildPushdownIntent_LikePrefixBecomesStartswith(t *testing.T) { - intent, ok := buildPushdownIntent(mustSelect(t, "select a from t where n like 'A%'")) - if !ok { - t.Fatalf("expected pushable intent") - } - preds := intent.GetPredicates() - if len(preds) != 1 { - t.Fatalf("preds = %v", preds) - } - p := preds[0] - if p.GetColumn() != "n" || p.GetOperator() != "startswith" || p.GetValue() != "A" { - t.Errorf("predicate = col=%s op=%s val=%v", p.GetColumn(), p.GetOperator(), p.GetValue()) - } -} - -func TestBuildPushdownIntent_LikeNonPrefixIsResidual(t *testing.T) { - // A non-prefix LIKE is not translatable; it must not appear in predicates. - intent, ok := buildPushdownIntent(mustSelect(t, "select a from t where n like '%A%'")) - if !ok { - t.Fatalf("expected an intent (projection present)") - } - if len(intent.GetPredicates()) != 0 { - t.Errorf("expected no pushable predicates, got %v", intent.GetPredicates()) - } -} - -func TestBuildPushdownIntent_CountStar(t *testing.T) { - intent, ok := buildPushdownIntent(mustSelect(t, "select count(*) as cnt from t")) - if !ok || !intent.IsCount() { - t.Errorf("expected Count=true, got ok=%v isCount=%v", ok, ok && intent.IsCount()) - } -} - -func TestBuildPushdownIntent_StarHasNoProjection(t *testing.T) { - intent, ok := buildPushdownIntent(mustSelect(t, "select * from t where x = 'v'")) - if !ok { - t.Fatalf("expected an intent (predicate present)") - } - if len(intent.GetProjection()) != 0 { - t.Errorf("expected no projection for SELECT *, got %v", intent.GetProjection()) - } -} - -func TestBuildPushdownIntent_JoinIsNotPushable(t *testing.T) { - // A join could under-fetch if a limit/offset were pushed to one leg. - _, ok := buildPushdownIntent(mustSelect(t, - "select a from t1 join t2 on t1.id = t2.id where t1.x = 'v' limit 5")) - if ok { - t.Errorf("expected join to be non-pushable") - } -} - -func TestBuildPushdownIntent_OrIsResidual(t *testing.T) { - // OR is not flattened into pushable AND-conjoined predicates. - intent, ok := buildPushdownIntent(mustSelect(t, "select a from t where x = 'v' or y = 'w'")) - if !ok { - t.Fatalf("expected an intent (projection present)") - } - if len(intent.GetPredicates()) != 0 { - t.Errorf("expected OR to be residual, got %v", intent.GetPredicates()) - } -} - -func TestBuildPushdownIntent_GrainChangeIsNotPushable(t *testing.T) { - // GROUP BY / DISTINCT / HAVING change grain: LIMIT/projection must stay client-side. - cases := []string{ - "select city, count(*) as c from t group by city limit 5", - "select distinct city from t limit 5", - "select city from t group by city having count(*) > 1", - } - for _, sql := range cases { - if _, ok := buildPushdownIntent(mustSelect(t, sql)); ok { - t.Errorf("expected non-pushable for %q", sql) - } - } -} - -func TestBuildPushdownIntent_ProjectionUnionsWhereAndOrderColumns(t *testing.T) { - // Issue #682: a WHERE-only column must reach the pushed projection or the - // authoritative client-side filter sees an absent column and drops all rows. - intent, ok := buildPushdownIntent(mustSelect(t, - "select id, displayName from users where userType = 'Member'")) - if !ok { - t.Fatalf("expected pushable intent") - } - proj := intent.GetProjection() - want := []string{"id", "displayName", "userType"} - if len(proj) != len(want) { - t.Fatalf("projection = %v, want %v", proj, want) - } - for i := range want { - if proj[i] != want[i] { - t.Errorf("projection[%d] = %s, want %s", i, proj[i], want[i]) - } - } -} - -func TestBuildPushdownIntent_ProjectionUnionIncludesResidualPredicateColumns(t *testing.T) { - // OR branches are not pushable predicates, but their columns are still needed - // client-side, so they must be fetched. - intent, ok := buildPushdownIntent(mustSelect(t, - "select a from t where x = 'v' or y = 'w'")) - if !ok { - t.Fatalf("expected an intent (projection present)") - } - if len(intent.GetPredicates()) != 0 { - t.Errorf("expected OR to remain residual, got %v", intent.GetPredicates()) - } - proj := intent.GetProjection() - want := []string{"a", "x", "y"} - if len(proj) != len(want) { - t.Fatalf("projection = %v, want %v", proj, want) - } - for i := range want { - if proj[i] != want[i] { - t.Errorf("projection[%d] = %s, want %s", i, proj[i], want[i]) - } - } -} - -func TestBuildPushdownIntent_ProjectionUnionDeduplicates(t *testing.T) { - intent, ok := buildPushdownIntent(mustSelect(t, - "select a, b from t where a = 'v' and b > 1 order by a asc")) - if !ok { - t.Fatalf("expected pushable intent") - } - proj := intent.GetProjection() - if len(proj) != 2 || proj[0] != "a" || proj[1] != "b" { - t.Errorf("expected deduplicated projection [a b], got %v", proj) - } -} - -func TestBuildPushdownIntent_CountSuppressesLimitAndProjection(t *testing.T) { - // A bare COUNT(*) pushes WHERE + count, but never top/skip/select/orderby. - intent, ok := buildPushdownIntent(mustSelect(t, - "select count(*) as cnt from t where x = 'v' order by y limit 5 offset 2")) - if !ok || !intent.IsCount() { - t.Fatalf("expected Count intent, got ok=%v", ok) - } - if len(intent.GetPredicates()) != 1 { - t.Errorf("expected the WHERE to still push, got %v", intent.GetPredicates()) - } - _, limSet := intent.GetLimit() - _, offSet := intent.GetOffset() - if limSet || offSet || len(intent.GetProjection()) != 0 || len(intent.GetOrderBy()) != 0 { - t.Errorf("expected top/skip/select/orderby suppressed for COUNT") - } -} diff --git a/internal/stackql/querysubmit/querysubmit.go b/internal/stackql/querysubmit/querysubmit.go index aecd39808..ee0ca752f 100644 --- a/internal/stackql/querysubmit/querysubmit.go +++ b/internal/stackql/querysubmit/querysubmit.go @@ -1,12 +1,12 @@ package querysubmit import ( - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/binlog" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/plan" "github.com/stackql/stackql/internal/stackql/planbuilder" "github.com/stackql/stackql/internal/stackql/primitivegraph" diff --git a/internal/stackql/registry/archive.go b/internal/stackql/registry/archive.go new file mode 100644 index 000000000..93f812599 --- /dev/null +++ b/internal/stackql/registry/archive.go @@ -0,0 +1,80 @@ +package registry + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "github.com/Masterminds/semver" +) + +func latestVersion(versions []string) (string, error) { + available := make([]*semver.Version, 0, len(versions)) + for _, v := range versions { + parsed, err := semver.NewVersion(v) + if err != nil { + return "", err + } + available = append(available, parsed) + } + if len(available) == 0 { + return "", fmt.Errorf("no versions available") + } + sort.Sort(semver.Collection(available)) + return available[len(available)-1].Original(), nil +} + +func decompressToPath(reader io.ReadCloser, target string) error { + defer reader.Close() + compressed, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer compressed.Close() + archive := tar.NewReader(compressed) + for { + header, nextErr := archive.Next() + if nextErr == io.EOF { + return nil + } + if nextErr != nil { + return nextErr + } + if entryErr := extractEntry(archive, header, target); entryErr != nil { + return entryErr + } + } +} + +func extractEntry(reader io.Reader, header *tar.Header, target string) error { + if !filepath.IsLocal(header.Name) { + return fmt.Errorf("invalid archive path %q", header.Name) + } + name := filepath.Join(target, header.Name) // #nosec G305 -- IsLocal rejects absolute paths and parent traversal. + if header.Typeflag == tar.TypeDir { + return os.MkdirAll(name, 0755) + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { + return fmt.Errorf("unsupported archive entry %q", header.Name) + } + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + return err + } + file, err := os.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, copyErr := io.CopyN(file, reader, header.Size) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} + +// LatestVersion selects the latest published semantic version from a registry entry. +func LatestVersion(versions []string) (string, error) { return latestVersion(versions) } diff --git a/internal/stackql/registry/document.go b/internal/stackql/registry/document.go new file mode 100644 index 000000000..7e1d87d2c --- /dev/null +++ b/internal/stackql/registry/document.go @@ -0,0 +1,52 @@ +package registry + +import ( + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/sqlengine" + "gopkg.in/yaml.v3" +) + +// ProviderDocument is the host's registration metadata; provider requests use Omni. +type ProviderDocument interface { + GetName() string + GetMinStackQLVersion() string + GetAuth() (*omnisdk.ProviderAuth, bool) + ExternalTables() (map[string]sqlengine.SQLExternalTable, error) +} + +type providerDocument struct { + Name string `yaml:"name"` + Config struct { + Auth *omnisdk.ProviderAuth `yaml:"auth"` + Minimum string `yaml:"minStackQLVersion"` + Tables map[string]yaml.Node `yaml:"sqlExternalTables"` + } `yaml:"config"` +} + +func parseProviderDocument(data []byte) (ProviderDocument, error) { + var doc providerDocument + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, err + } + return &doc, nil +} +func (d *providerDocument) GetName() string { return d.Name } +func (d *providerDocument) GetMinStackQLVersion() string { return d.Config.Minimum } +func (d *providerDocument) GetAuth() (*omnisdk.ProviderAuth, bool) { + return d.Config.Auth, d.Config.Auth != nil +} +func (d *providerDocument) ExternalTables() (map[string]sqlengine.SQLExternalTable, error) { + out := make(map[string]sqlengine.SQLExternalTable, len(d.Config.Tables)) + for name, node := range d.Config.Tables { + data, err := yaml.Marshal(&node) + if err != nil { + return nil, err + } + table, err := sqlengine.ExternalTable(data) + if err != nil { + return nil, err + } + out[name] = table + } + return out, nil +} diff --git a/internal/stackql/registry/registry.go b/internal/stackql/registry/registry.go new file mode 100644 index 000000000..9880e9803 --- /dev/null +++ b/internal/stackql/registry/registry.go @@ -0,0 +1,587 @@ +package registry + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-provider-registry/signing/pqsig" + "gopkg.in/yaml.v3" +) + +const ( + defaultRegistryUrlString = "https://raw.githubusercontent.com/stackql/" + + "stackql-provider-registry/intial-devel/providers" + defaultSrcPrefix string = "src" + defaultDistPrefix string = "dist" + httpSchemeRegexpString string = `(?i)^https?$` + fileSchemeRegexpString string = `(?i)^file$` + remoteProviderListPath string = `providers.yaml` +) + +var ( + httpSchemeRegexp *regexp.Regexp = regexp.MustCompile(httpSchemeRegexpString) + fileSchemeRegexp *regexp.Regexp = regexp.MustCompile(fileSchemeRegexpString) +) + +type RegistryAPI interface { + PullAndPersistProviderArchive(string, string) error + ListAllAvailableProviders() (map[string]ProviderDescription, error) + ListAllProviderVersions(string) (map[string]ProviderDescription, error) + ListLocallyAvailableProviders() map[string]ProviderDescription + GetDocBytes(string) ([]byte, error) + GetLatestAvailableVersion(string) (string, error) + GetLatestPublishedVersion(string) (string, error) + LoadProviderByName(string, string) (ProviderDocument, error) + RemoveProviderVersion(string, string) error + ClearProviderCache(string) error + GetLocalProviderDocPath(string, string) string +} + +type RegistryConfig = omnisdk.RegistryConfig + +type standardRegistry struct { + allowSrcDownload bool + regUrl *url.URL + srcUrl *url.URL + distUrl *url.URL + localDocRoot string + localSrcPrefix string + localDistPrefix string + transport http.RoundTripper + verifier pqsig.Verifier + nopVerifier bool +} + +func NewRegistry(registryCfg RegistryConfig, transport http.RoundTripper) (RegistryAPI, error) { + return newRegistry(registryCfg, transport) +} + +func newRegistry(registryCfg RegistryConfig, transport http.RoundTripper) (RegistryAPI, error) { + registryUrl := registryCfg.RegistryURL + if registryUrl == "" { + registryUrl = defaultRegistryUrlString + } + srcUrlStr := registryUrl + var srcPrefix string + if registryCfg.SrcPrefix == nil { + srcPrefix = defaultSrcPrefix + } else { + srcPrefix = *registryCfg.SrcPrefix + } + if srcPrefix != "" { + srcUrlStr = fmt.Sprintf("%s/%s", registryUrl, srcPrefix) + } + distUrlStr := registryUrl + var distPrefix string + if registryCfg.DistPrefix == nil { + distPrefix = defaultDistPrefix + } else { + distPrefix = *registryCfg.DistPrefix + } + if distPrefix != "" { + distUrlStr = fmt.Sprintf("%s/%s", registryUrl, distPrefix) + } + regUrl, err := url.Parse(registryUrl) + if err != nil { + return nil, err + } + srcUrl, err := url.Parse(srcUrlStr) + if err != nil { + return nil, err + } + distUrl, err := url.Parse(distUrlStr) + if err != nil { + return nil, err + } + var ver pqsig.Verifier + nopVerify := false + var keys map[string]string + if registryCfg.VerifyConfig != nil { + nopVerify = registryCfg.VerifyConfig.NopVerify + keys = registryCfg.VerifyConfig.PublicKeys + } + if !nopVerify { + ver, err = pqsig.NewVerifier(keys) + if err != nil { + return nil, err + } + } + rv := &standardRegistry{ + allowSrcDownload: registryCfg.AllowSrcDownload, + regUrl: regUrl, + srcUrl: srcUrl, + distUrl: distUrl, + localDocRoot: registryCfg.LocalDocRoot, + localSrcPrefix: srcPrefix, + localDistPrefix: distPrefix, + transport: transport, + verifier: ver, + nopVerifier: nopVerify, + } + return rv, nil +} + +func (r *standardRegistry) GetLocalDocRoot() string { + return r.getLocalDocRoot() +} + +func (r *standardRegistry) ListLocallyAvailableProviders() map[string]ProviderDescription { + return r.listLocalProviders() +} + +type ProviderDescription = omnisdk.ProviderDescription + +type ProvidersList struct { + _ struct{} + Providers map[string]ProviderDescription `json:"providers" yaml:"providers"` +} + +func NewProvidersList() ProvidersList { + return ProvidersList{ + Providers: make(map[string]ProviderDescription), + } +} + +func (pl ProvidersList) GetLatestList() (ProvidersList, error) { + m := make(map[string]ProviderDescription) + for k, v := range pl.Providers { + latest, err := latestVersion(v.Versions) + if err != nil { + return NewProvidersList(), err + } + m[k] = ProviderDescription{Versions: []string{latest}} + } + return ProvidersList{Providers: m}, nil +} + +func (pl ProvidersList) GetSingleProviderList(prov string) ProvidersList { + m := make(map[string]ProviderDescription) + for k, v := range pl.Providers { + if k == prov { + m[k] = v + } + } + return ProvidersList{Providers: m} +} + +func (r *standardRegistry) ListAllAvailableProviders() (map[string]ProviderDescription, error) { + if r.isFile() { + return nil, fmt.Errorf("'registry list' is meaningless in local mode") + } + regProvs := NewProvidersList() + rc, err := r.getRemoteProviderList() + if err != nil { + return nil, err + } + b, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(b, ®Provs) + if err != nil { + return nil, err + } + latest, err := regProvs.GetLatestList() + if err != nil { + return nil, err + } + return latest.Providers, nil +} + +func (r *standardRegistry) ListAllProviderVersions(prov string) (map[string]ProviderDescription, error) { + return r.listAllProviderVersions(prov) +} + +func (r *standardRegistry) listAllProviderVersions(prov string) (map[string]ProviderDescription, error) { + if r.isFile() { + return nil, fmt.Errorf("'registry list' is meaningless in local mode") + } + regProvs := NewProvidersList() + rc, err := r.getRemoteProviderList() + if err != nil { + return nil, err + } + b, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(b, ®Provs) + if err != nil { + return nil, err + } + singleProvList := regProvs.GetSingleProviderList(prov) + return singleProvList.Providers, nil +} + +func (r *standardRegistry) isHttp() bool { + return httpSchemeRegexp.MatchString(r.regUrl.Scheme) +} + +func (r *standardRegistry) isFile() bool { + return fileSchemeRegexp.MatchString(r.regUrl.Scheme) +} + +func (r *standardRegistry) isLocalFile() bool { + return r.isFile() && strings.HasPrefix(r.regUrl.Path, "/") +} + +func (r *standardRegistry) GetDocBytes(docPath string) ([]byte, error) { + return r.getVerifiedDocBytes(docPath) +} + +func (r *standardRegistry) getProviderDocBytes(prov string, version string) ([]byte, error) { + prov = r.renameProvider(prov) + return r.getVerifiedDocBytes(path.Join(prov, version, "provider.yaml")) +} + +func (r *standardRegistry) PullProviderArchive(prov string, version string) (io.ReadCloser, error) { + return r.pullProviderArchive(prov, version) +} + +func (r *standardRegistry) pullProviderArchive(prov string, version string) (io.ReadCloser, error) { + prov = r.renameProvider(prov) + fp := path.Join(prov, fmt.Sprintf("%s.tgz", version)) + return r.pullArchive(fp) +} + +func (r *standardRegistry) PullAndPersistProviderArchive(prov string, version string) error { + return r.pullAndPersistProviderArchive(prov, version) +} + +func (r *standardRegistry) pullAndPersistProviderArchive(prov string, version string) error { + if r.localDocRoot == "" { + return fmt.Errorf("cannot pull provider without local doc location") + } + rdr, err := r.pullProviderArchive(prov, version) + if err != nil { + return err + } + pr := r.renameProvider(prov) + err = os.RemoveAll(path.Join(r.getLocalDocRoot(), pr, version)) + if err != nil { + return err + } + return decompressToPath(rdr, path.Join(r.getLocalDocRoot(), pr)) +} + +func (r *standardRegistry) LoadProviderByName(prov, version string) (ProviderDocument, error) { + if version == "" { + var err error + version, err = r.GetLatestAvailableVersion(prov) + if err != nil { + return nil, err + } + } + b, err := r.getProviderDocBytes(prov, version) + if err != nil { + return nil, err + } + return parseProviderDocument(b) +} + +func (r *standardRegistry) pullArchive(archivePath string) (io.ReadCloser, error) { + return r.getUnVerifiedArchive(archivePath) +} + +func (r *standardRegistry) getRemoteDoc(docPath string) (io.ReadCloser, error) { + cl := &http.Client{} + if r.transport != nil { + cl.Transport = r.transport + } + response, err := cl.Get(fmt.Sprintf("%s/%s", r.srcUrl.String(), docPath)) + if err != nil { + return nil, err + } + if response.Body == nil { + return nil, fmt.Errorf("no response body from remote") + } + return response.Body, nil +} + +func (r *standardRegistry) getRemoteArchive(docPath string) (io.ReadCloser, error) { + cl := &http.Client{} + if r.transport != nil { + cl.Transport = r.transport + } + response, err := cl.Get(fmt.Sprintf("%s/%s", r.distUrl.String(), docPath)) + if err != nil { + return nil, err + } + if response.Body == nil { + return nil, fmt.Errorf("no response body from remote") + } + return response.Body, nil +} + +func (r *standardRegistry) getRemoteProviderList() (io.ReadCloser, error) { + cl := &http.Client{} + if r.transport != nil { + cl.Transport = r.transport + } + response, err := cl.Get(fmt.Sprintf("%s/%s", r.distUrl.String(), remoteProviderListPath)) + if err != nil { + return nil, err + } + if response.Body == nil { + return nil, fmt.Errorf("no response body from remote") + } + return response.Body, nil +} + +func (r *standardRegistry) GetLocalDocTrunk() string { + return r.getLocalDocTrunk() +} + +func (r *standardRegistry) getLocalDocTrunk() string { + return r.localDocRoot +} + +func (r *standardRegistry) getLocalDocRoot() string { + switch r.localSrcPrefix { + case "": + return r.localDocRoot + default: + return path.Join(r.localDocRoot, r.localSrcPrefix) + } +} + +type ProviderInfo struct { + Name string + Version string +} + +//nolint:gocognit // Preserve the registry directory traversal and filtering. +func (r *standardRegistry) listLocalProviders() map[string]ProviderDescription { + dr := r.getLocalDocRoot() + switch dr { + case "": + return map[string]ProviderDescription{} + default: + provs, err := os.ReadDir(dr) + if err != nil { + return map[string]ProviderDescription{} + } + rv := make(map[string]ProviderDescription) + for _, p := range provs { + if p.IsDir() && !strings.HasPrefix(p.Name(), ".") { + versions, versionErr := os.ReadDir(path.Join(dr, p.Name())) + var val ProviderDescription + if versionErr == nil { + for _, v := range versions { + if v.IsDir() && !strings.HasPrefix(v.Name(), ".") { + val.Versions = append(val.Versions, v.Name()) + } + } + } + rv[p.Name()] = val + } + } + return rv + } +} + +func (r *standardRegistry) getLocalArchiveRoot() string { + switch r.localDistPrefix { + case "": + return r.localDocRoot + default: + return path.Join(r.localDocRoot, r.localDistPrefix) + } +} + +func (r *standardRegistry) renameProvider(prov string) string { + if prov == "google" { + prov = "googleapis.com" + } + return prov +} + +func (r *standardRegistry) GetLocalProviderDocPath(prov string, version string) string { + return r.getLocalProviderDocPath(prov, version) +} + +func (r *standardRegistry) getLocalProviderDocPath(prov string, version string) string { + return path.Join(r.getLocalProviderRootDirPath(prov, version), "provider.yaml") +} + +func (r *standardRegistry) getLocalProviderRootDirPath(prov string, version string) string { + prov = r.renameProvider(prov) + return path.Join(r.getLocalDocRoot(), prov, version) +} + +func (r *standardRegistry) getLocalDocPath(docPath string) string { + return path.Join(r.getLocalDocRoot(), docPath) +} + +func (r *standardRegistry) getLocalArchivePath(docPath string) string { + return path.Join(r.getLocalArchiveRoot(), docPath) +} + +func (r *standardRegistry) getLocalDoc(docPath string) (io.ReadCloser, error) { + // localPath := r.getLocalDocPath(docPath) + fi, err := os.Open(docPath) + if fi != nil { + defer fi.Close() + } else { + return nil, fmt.Errorf("nil file") + } + if err != nil { + return nil, err + } + fb, readErr := io.ReadAll(fi) + if readErr != nil { + return nil, fmt.Errorf("cannot read local file: '%s'", readErr.Error()) + } + return io.NopCloser(bytes.NewReader(fb)), nil +} + +func (r *standardRegistry) getUnVerifiedArchive(docPath string) (io.ReadCloser, error) { + if r.isLocalFile() { + return os.Open(path.Join(r.distUrl.Path, docPath)) + } + if r.localDocRoot != "" { + localPath := r.getLocalArchivePath(docPath) + lf, err := r.getLocalDoc(localPath) + if err == nil { + return lf, nil + } + } + if r.isHttp() { + return r.getRemoteArchive(docPath) + } + return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme) +} + +func (r *standardRegistry) getDocReader(docPath string) (io.ReadCloser, error) { + if r.isLocalFile() { + return os.Open(path.Join(r.localSourcePath(), docPath)) + } + if r.localDocRoot != "" { + localPath := r.getLocalDocPath(docPath) + lf, err := r.getLocalDoc(localPath) + if err == nil { + return lf, nil + } + } + if r.isHttp() { + if !r.allowSrcDownload { + return nil, fmt.Errorf("download of individual docs disallowed; please attempt to pull provider docs") + } + response, err := r.getRemoteDoc(docPath) + if err != nil { + return nil, err + } + if response == nil { + return nil, fmt.Errorf("no response body from remote") + } + return response, nil + } + return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme) +} + +func (r *standardRegistry) getVerifiedDocBytes(docPath string) ([]byte, error) { + reader, err := r.getDocReader(docPath) + if err != nil { + return nil, fmt.Errorf("registry document %q: %w", docPath, err) + } + defer reader.Close() + doc, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read registry document %q: %w", docPath, err) + } + if r.nopVerifier { + return doc, nil + } + doc, err = r.verifier.Verify(doc) + if err != nil { + return nil, fmt.Errorf("verify registry document %q: %w", docPath, err) + } + return doc, nil +} + +func (r *standardRegistry) GetLatestAvailableVersion(providerName string) (string, error) { + return r.getLatestAvailableVersion(providerName) +} + +func (r *standardRegistry) localSourcePath() string { + localPath := r.srcUrl.Path + if r.srcUrl.Host == "." { + localPath = "." + localPath + } + if strings.HasPrefix(localPath, "/") && filepath.VolumeName(localPath[1:]) != "" { + localPath = localPath[1:] + } + return localPath +} + +func (r *standardRegistry) getLatestAvailableVersion(providerName string) (string, error) { + providerName = r.renameProvider(providerName) + if r.isLocalFile() { + deSlice, err := os.ReadDir(path.Join(r.localSourcePath(), providerName)) + if err != nil { + return "", err + } + var deStrSlice []string + for _, e := range deSlice { + if e.IsDir() { + deStrSlice = append(deStrSlice, e.Name()) + } + } + return latestVersion(deStrSlice) + } + + deSlice, err := os.ReadDir(path.Join(r.getLocalDocRoot(), providerName)) + if err != nil { + return "", err + } + var deStrSlice []string + for _, e := range deSlice { + if e.IsDir() { + deStrSlice = append(deStrSlice, e.Name()) + } + } + return latestVersion(deStrSlice) +} + +func (r *standardRegistry) GetLatestPublishedVersion(providerName string) (string, error) { + return r.getLatestPublishedVersion(providerName) +} + +func (r *standardRegistry) getLatestPublishedVersion(providerName string) (string, error) { + p, err := r.listAllProviderVersions(providerName) + if err != nil { + return "", err + } + description, ok := p[providerName] + if !ok { + return "", fmt.Errorf("could not resolve provider '%s'", providerName) + } + latestVersion, err := latestVersion(description.Versions) + if err != nil { + return "", err + } + return latestVersion, nil +} + +// RemoveProviderVersion removes a specific version of a provider +// For example: RemoveProviderVersion("okta", "v23.09.00169"). +func (r *standardRegistry) RemoveProviderVersion(providerId string, version string) error { + providerPath := r.getLocalProviderRootDirPath(providerId, version) + return os.RemoveAll(providerPath) +} + +// ClearProviderCache clears the cache for a specific provider +// For example: ClearProviderCache("aws"). +func (r *standardRegistry) ClearProviderCache(providerId string) error { + cachePath := path.Join(r.getLocalDocRoot(), providerId) + return os.RemoveAll(cachePath) +} diff --git a/internal/stackql/registry/registry_test.go b/internal/stackql/registry/registry_test.go new file mode 100644 index 000000000..cf81936b9 --- /dev/null +++ b/internal/stackql/registry/registry_test.go @@ -0,0 +1,74 @@ +package registry_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-provider-registry/signing/pqsig" + "github.com/stackql/stackql/internal/stackql/registry" +) + +func TestLocalRegistrationKeepsHostSQLAndCredentialDefaults(t *testing.T) { + root := t.TempDir() + providerDir := filepath.Join(root, "src", "demo", "v1.2.3") + if err := os.MkdirAll(providerDir, 0755); err != nil { + t.Fatal(err) + } + const doc = `name: demo +config: + minStackQLVersion: v1.0.0 + auth: + type: bearer + credentialsenvvar: DEMO_TOKEN + successor: + type: null + sqlExternalTables: + widgets: + catalogName: catalog + schemaName: public + name: widgets + columns: + - name: id + type: integer + oid: 23 +` + if err := os.WriteFile(filepath.Join(providerDir, "provider.yaml"), []byte(doc), 0600); err != nil { + t.Fatal(err) + } + reg, err := registry.NewRegistry(omnisdk.RegistryConfig{LocalDocRoot: root, VerifyConfig: &pqsig.VerifierConfig{NopVerify: true}}, nil) + if err != nil { + t.Fatal(err) + } + available := reg.ListLocallyAvailableProviders() + if len(available["demo"].Versions) != 1 { + t.Fatalf("missing local provider: %v", available) + } + loaded, err := reg.LoadProviderByName("demo", "") + if err != nil { + t.Fatal(err) + } + if loaded.GetName() != "demo" || loaded.GetMinStackQLVersion() != "v1.0.0" { + t.Fatal("lost registration metadata") + } + auth, ok := loaded.GetAuth() + if !ok || auth.KeyEnvVar != "DEMO_TOKEN" || auth.Successor == nil { + t.Fatal("lost credential defaults") + } + tables, err := loaded.ExternalTables() + if err != nil { + t.Fatal(err) + } + table := tables["widgets"] + if table.GetName() != "widgets" || table.GetSchemaName() != "public" || len(table.GetColumns()) != 1 || table.GetColumns()[0].GetOid() != 23 { + t.Fatal("lost external SQL metadata") + } +} + +func TestLatestVersionUsesSemanticOrdering(t *testing.T) { + latest, err := registry.LatestVersion([]string{"v0.13.9", "v0.13.100", "v0.13.10"}) + if err != nil || latest != "v0.13.100" { + t.Fatalf("latest=%q err=%v", latest, err) + } +} diff --git a/internal/stackql/relational/predicate.go b/internal/stackql/relational/predicate.go deleted file mode 100644 index 58f445e3b..000000000 --- a/internal/stackql/relational/predicate.go +++ /dev/null @@ -1,125 +0,0 @@ -package relational - -import ( - "fmt" - "regexp" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/sqltypes" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql-parser/go/vt/vtgate/evalengine" -) - -func AndTableFilters( - lhs, - rhs func(formulation.ITable) (formulation.ITable, error), -) func(formulation.ITable) (formulation.ITable, error) { - if lhs == nil { - return rhs - } - return func(t formulation.ITable) (formulation.ITable, error) { - lResult, lErr := lhs(t) - rResult, rErr := rhs(t) - if lErr != nil { - return nil, lErr - } - if rErr != nil { - return nil, rErr - } - if lResult != nil && rResult != nil { - return lResult, nil - } - return nil, nil - } -} - -func OrTableFilters( - lhs, - rhs func(formulation.ITable) (formulation.ITable, error), -) func(formulation.ITable) (formulation.ITable, error) { - if lhs == nil { - return rhs - } - return func(t formulation.ITable) (formulation.ITable, error) { - lResult, lErr := lhs(t) - rResult, rErr := rhs(t) - if lErr != nil { - return nil, lErr - } - if rErr != nil { - return nil, rErr - } - if lResult != nil { - return lResult, nil - } - if rResult != nil { - return rResult, nil - } - return nil, nil - } -} - -func ConstructTablePredicateFilter( - colName string, rhs sqltypes.Value, - operatorPredicate func(int) bool) func(formulation.ITable) (formulation.ITable, error) { - return func(row formulation.ITable) (formulation.ITable, error) { - v, e := row.GetKeyAsSqlVal(colName) - if e != nil { - return nil, e - } - result, err := evalengine.NullsafeCompare(v, rhs) - if err == nil && operatorPredicate(result) { - return row, nil - } - return nil, err - } -} - -func ConstructLikePredicateFilter( - colName string, - rhs *regexp.Regexp, isNegating bool) func(formulation.ITable) (formulation.ITable, error) { - return func(row formulation.ITable) (formulation.ITable, error) { - v, vErr := row.GetKey(colName) - if vErr != nil { - return nil, vErr - } - s, sOk := v.(string) - if !sOk { - return nil, fmt.Errorf("cannot compare non-string type '%T' with regex", v) - } - if rhs.MatchString(s) != isNegating { - return row, nil - } - return nil, nil - } -} - -func GetOperatorPredicate(operator string) (func(int) bool, error) { - switch operator { - case sqlparser.EqualStr: - return func(result int) bool { - return result == 0 - }, nil - case sqlparser.NotEqualStr: - return func(result int) bool { - return result != 0 - }, nil - case sqlparser.GreaterEqualStr: - return func(result int) bool { - return result >= 0 - }, nil - case sqlparser.GreaterThanStr: - return func(result int) bool { - return result > 0 - }, nil - case sqlparser.LessEqualStr: - return func(result int) bool { - return result <= 0 - }, nil - case sqlparser.LessThanStr: - return func(result int) bool { - return result < 0 - }, nil - } - return nil, fmt.Errorf("cannot determine predicate") -} diff --git a/internal/stackql/relational/predicate_test.go b/internal/stackql/relational/predicate_test.go deleted file mode 100644 index 65d9da7da..000000000 --- a/internal/stackql/relational/predicate_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package relational_test - -import ( - "errors" - "regexp" - "testing" - - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/sqltypes" - "github.com/stackql/stackql/internal/stackql/relational" -) - -var ( - errTestTable = errors.New("test table error") - errLHS = errors.New("lhs error") - errRHS = errors.New("rhs error") -) - -type testTable struct { - name string - values map[string]any - sqlValues map[string]sqltypes.Value - err error -} - -func (t *testTable) GetKey(key string) (any, error) { - if t.err != nil { - return nil, t.err - } - value, ok := t.values[key] - if !ok { - return nil, errTestTable - } - return value, nil -} - -func (t *testTable) GetKeyAsSqlVal(key string) (sqltypes.Value, error) { - if t.err != nil { - return sqltypes.NULL, t.err - } - value, ok := t.sqlValues[key] - if !ok { - return sqltypes.NULL, errTestTable - } - return value, nil -} - -func (t *testTable) GetName() string { return t.name } -func (t *testTable) KeyExists(key string) bool { _, ok := t.values[key]; return ok } - -func tableFilter(result formulation.ITable, err error) func(formulation.ITable) (formulation.ITable, error) { - return func(formulation.ITable) (formulation.ITable, error) { - return result, err - } -} - -func TestAndTableFilters(t *testing.T) { - row := &testTable{name: "row"} - lhsResult := &testTable{name: "lhs"} - rhsResult := &testTable{name: "rhs"} - rhs := tableFilter(row, nil) - if got, err := relational.AndTableFilters(nil, rhs)(row); err != nil || got != row { - t.Fatalf("nil lhs should return rhs result: got %v, err %v", got, err) - } - - tests := []struct { - name string - lhs func(formulation.ITable) (formulation.ITable, error) - rhs func(formulation.ITable) (formulation.ITable, error) - want formulation.ITable - wantErr error - }{ - {name: "both match returns lhs", lhs: tableFilter(lhsResult, nil), rhs: tableFilter(rhsResult, nil), want: lhsResult}, - {name: "lhs misses", lhs: tableFilter(nil, nil), rhs: tableFilter(row, nil)}, - {name: "rhs misses", lhs: tableFilter(row, nil), rhs: tableFilter(nil, nil)}, - {name: "lhs error", lhs: tableFilter(nil, errLHS), rhs: tableFilter(row, nil), wantErr: errLHS}, - {name: "rhs error", lhs: tableFilter(row, nil), rhs: tableFilter(nil, errRHS), wantErr: errRHS}, - {name: "lhs error takes precedence", lhs: tableFilter(nil, errLHS), rhs: tableFilter(nil, errRHS), wantErr: errLHS}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := relational.AndTableFilters(tt.lhs, tt.rhs)(row) - if !errors.Is(err, tt.wantErr) || got != tt.want { - t.Fatalf("got (%v, %v), want (%v, %v)", got, err, tt.want, tt.wantErr) - } - }) - } -} - -func TestOrTableFilters(t *testing.T) { - row := &testTable{name: "row"} - lhsResult := &testTable{name: "lhs"} - rhsResult := &testTable{name: "rhs"} - rhs := tableFilter(row, nil) - if got, err := relational.OrTableFilters(nil, rhs)(row); err != nil || got != row { - t.Fatalf("nil lhs should return rhs result: got %v, err %v", got, err) - } - - tests := []struct { - name string - lhs func(formulation.ITable) (formulation.ITable, error) - rhs func(formulation.ITable) (formulation.ITable, error) - want formulation.ITable - wantErr error - }{ - {name: "lhs matches", lhs: tableFilter(lhsResult, nil), rhs: tableFilter(nil, nil), want: lhsResult}, - {name: "rhs matches", lhs: tableFilter(nil, nil), rhs: tableFilter(rhsResult, nil), want: rhsResult}, - {name: "both match returns lhs", lhs: tableFilter(lhsResult, nil), rhs: tableFilter(rhsResult, nil), want: lhsResult}, - {name: "neither matches", lhs: tableFilter(nil, nil), rhs: tableFilter(nil, nil)}, - {name: "lhs error", lhs: tableFilter(nil, errLHS), rhs: tableFilter(row, nil), wantErr: errLHS}, - {name: "rhs error", lhs: tableFilter(row, nil), rhs: tableFilter(nil, errRHS), wantErr: errRHS}, - {name: "lhs error takes precedence", lhs: tableFilter(nil, errLHS), rhs: tableFilter(nil, errRHS), wantErr: errLHS}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := relational.OrTableFilters(tt.lhs, tt.rhs)(row) - if !errors.Is(err, tt.wantErr) || got != tt.want { - t.Fatalf("got (%v, %v), want (%v, %v)", got, err, tt.want, tt.wantErr) - } - }) - } -} - -func TestConstructTablePredicateFilter(t *testing.T) { - row := &testTable{sqlValues: map[string]sqltypes.Value{ - "age": sqltypes.NewInt64(42), - "expression": sqltypes.MakeTrusted(sqltypes.Expression, []byte("x")), - }} - tests := []struct { - name string - operator string - rhs sqltypes.Value - want formulation.ITable - }{ - {name: "equal", operator: "=", rhs: sqltypes.NewInt64(42), want: row}, - {name: "not equal", operator: "!=", rhs: sqltypes.NewInt64(41), want: row}, - {name: "greater than", operator: ">", rhs: sqltypes.NewInt64(41), want: row}, - {name: "greater than rejects larger rhs", operator: ">", rhs: sqltypes.NewInt64(43)}, - {name: "greater than or equal", operator: ">=", rhs: sqltypes.NewInt64(42), want: row}, - {name: "less than", operator: "<", rhs: sqltypes.NewInt64(43), want: row}, - {name: "less than or equal", operator: "<=", rhs: sqltypes.NewInt64(42), want: row}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - predicate, err := relational.GetOperatorPredicate(tt.operator) - if err != nil { - t.Fatal(err) - } - got, err := relational.ConstructTablePredicateFilter("age", tt.rhs, predicate)(row) - if err != nil || got != tt.want { - t.Fatalf("got (%v, %v), want (%v, nil)", got, err, tt.want) - } - }) - } - - equal, err := relational.GetOperatorPredicate("=") - if err != nil { - t.Fatal(err) - } - if _, err = relational.ConstructTablePredicateFilter("missing", sqltypes.NewInt64(42), equal)(row); !errors.Is(err, errTestTable) { - t.Fatalf("missing column error = %v, want %v", err, errTestTable) - } - if _, err = relational.ConstructTablePredicateFilter("expression", sqltypes.MakeTrusted(sqltypes.Expression, []byte("x")), equal)(row); err == nil { - t.Fatal("uncomparable values should return an error") - } -} - -func TestConstructLikePredicateFilter(t *testing.T) { - re := regexp.MustCompile(`^Stack`) - row := &testTable{values: map[string]any{"name": "StackQL", "other": "SQL", "count": 3}} - - tests := []struct { - name string - column string - negating bool - want formulation.ITable - wantErr bool - }{ - {name: "like match", column: "name", want: row}, - {name: "like rejects nonmatch", column: "other"}, - {name: "not like rejects match", column: "name", negating: true}, - {name: "not like accepts nonmatch", column: "other", negating: true, want: row}, - {name: "missing column", column: "missing", wantErr: true}, - {name: "non string", column: "count", wantErr: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := relational.ConstructLikePredicateFilter(tt.column, re, tt.negating)(row) - if (err != nil) != tt.wantErr || got != tt.want { - t.Fatalf("got (%v, %v), want (%v, error=%v)", got, err, tt.want, tt.wantErr) - } - }) - } -} - -func TestGetOperatorPredicate(t *testing.T) { - tests := []struct { - operator string - matches []int - }{ - {operator: "=", matches: []int{0}}, - {operator: "!=", matches: []int{-1, 1}}, - {operator: ">=", matches: []int{0, 1}}, - {operator: ">", matches: []int{1}}, - {operator: "<=", matches: []int{-1, 0}}, - {operator: "<", matches: []int{-1}}, - } - for _, tt := range tests { - t.Run(tt.operator, func(t *testing.T) { - predicate, err := relational.GetOperatorPredicate(tt.operator) - if err != nil { - t.Fatal(err) - } - for _, result := range []int{-1, 0, 1} { - want := false - for _, match := range tt.matches { - want = want || result == match - } - if got := predicate(result); got != want { - t.Errorf("predicate(%d) = %v, want %v", result, got, want) - } - } - }) - } - if _, err := relational.GetOperatorPredicate("LIKE"); err == nil { - t.Fatal("unsupported operator should return an error") - } -} diff --git a/internal/stackql/router/combination_composer.go b/internal/stackql/router/combination_composer.go deleted file mode 100644 index 0411ca282..000000000 --- a/internal/stackql/router/combination_composer.go +++ /dev/null @@ -1,76 +0,0 @@ -package router - -var ( - _ combinationComposer = &standardCombinationComposer{} -) - -type parameterPack map[string][]any - -type combinationComposer interface { - analyse(parameterPack) error - getCombinations() []map[string]any - getIsAnythingSplit() bool -} - -type standardCombinationComposer struct { - isAnythingSplit bool - combinations []map[string]any -} - -func newCombinationComposer() combinationComposer { - return &standardCombinationComposer{} -} - -func (sc *standardCombinationComposer) getCombinations() []map[string]any { - return sc.combinations -} - -func (sc *standardCombinationComposer) getIsAnythingSplit() bool { - return sc.isAnythingSplit -} - -func (sc *standardCombinationComposer) getPi(pp parameterPack, keys []string) int { - var pi int = 1 //nolint:revive,stylecheck // prefer explicit - for _, k := range keys { - pi *= len(pp[k]) - } - return pi -} - -func (sc *standardCombinationComposer) getRotation(x int, piMMinusOne int) int { - return x / piMMinusOne // this is floor division for golang int type -} - -func (sc *standardCombinationComposer) getModularOrdinal(x int, piM int, piMMinusOne int) int { - // demi rotation modulo floor quotient of pi of M and pi of M exlusive of the current parameter - return sc.getRotation(x, piMMinusOne) % (piM / piMMinusOne) -} - -func (sc *standardCombinationComposer) analyse( - pp parameterPack, -) error { - var keyOrdering []string - for k := range pp { - keyOrdering = append(keyOrdering, k) - } - totalCombinationCount := sc.getPi(pp, keyOrdering) - if totalCombinationCount > 1 { - sc.isAnythingSplit = true - } - var modularOrdinals []map[string]int - for x := 0; x < totalCombinationCount; x++ { - modularOrdinalIter := make(map[string]int) - for i, key := range keyOrdering { - modularOrdinalIter[key] = sc.getModularOrdinal(x, sc.getPi(pp, keyOrdering[:i+1]), sc.getPi(pp, keyOrdering[:i])) - } - modularOrdinals = append(modularOrdinals, modularOrdinalIter) - } - for _, mo := range modularOrdinals { - combination := make(map[string]any) - for k, v := range mo { - combination[k] = pp[k][v] - } - sc.combinations = append(sc.combinations, combination) - } - return nil -} diff --git a/internal/stackql/router/obtain_context/obtain_context.go b/internal/stackql/router/obtain_context/obtain_context.go deleted file mode 100644 index 508464675..000000000 --- a/internal/stackql/router/obtain_context/obtain_context.go +++ /dev/null @@ -1,65 +0,0 @@ -package obtain_context //nolint:revive,cyclop,stylecheck // TODO: allow - -import ( - "fmt" - - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/sql_system" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" -) - -func ObtainAnnotationCtx( - sqlSystem sql_system.SQLSystem, - tbl tablemetadata.ExtendedTableMetadata, - parameters map[string]interface{}, - namespaceCollection tablenamespace.Collection, -) (taxonomy.AnnotationCtx, error) { - _, isView := tbl.GetView() - _, isSQLDataSource := tbl.GetSQLDataSource() - _, isSubquery := tbl.GetSubquery() - isPGInternalObject := tbl.GetHeirarchyObjects().IsPGInternalObject() - // Note: CTEs are converted to subqueries at AST level, so isSubquery handles them. - if isView || isSQLDataSource || isSubquery || isPGInternalObject { - // TODO: upgrade this flow; nil == YUCK!!! - return taxonomy.NewStaticStandardAnnotationCtx( - nil, tbl.GetHeirarchyObjects().GetHeirarchyIDs(), - tbl, parameters, false, - ), nil - } - schema, mediaType, err := tbl.GetResponseSchemaAndMediaType() - if err != nil { - return nil, err - } - // TODO: emulate in address space - itemObjS, selectItemsKey, err := schema.GetSelectSchema(tbl.LookupSelectItemsKey(), mediaType) - unsuitableSchemaMsg := "schema unsuitable for select query" - if err != nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) - } - tbl.SetSelectItemsKey(selectItemsKey) - provStr, _ := tbl.GetProviderStr() - svcStr, _ := tbl.GetServiceStr() - rscStr, _ := tbl.GetResourceStr() - methodStr, _ := tbl.GetMethodStr() - if itemObjS == nil { - return nil, fmt.Errorf("%s", unsuitableSchemaMsg) - } - name := itemObjS.GetSelectionName() - tbl, err = tbl.WithResponseSchemaStr(name) - if err != nil { - return nil, fmt.Errorf("%s: %w", unsuitableSchemaMsg, err) - } - tn, err := tbl.GetTableName() - if err == nil && namespaceCollection.GetAnalyticsCacheTableNamespaceConfigurator().IsAllowed(tn) { - name, _ = tbl.GetResponseSchemaStr() - } - hIDs := internaldto.NewHeirarchyIdentifiers(provStr, svcStr, rscStr, methodStr).WithResponseSchemaStr(name) - viewDTO, isView := sqlSystem.GetViewByName(hIDs.GetTableName()) - // TODO: match on params - if isView { - hIDs = hIDs.WithView(viewDTO) - } - return taxonomy.NewStaticStandardAnnotationCtx(itemObjS, hIDs, tbl, parameters, false), nil -} diff --git a/internal/stackql/router/parameter_router.go b/internal/stackql/router/parameter_router.go deleted file mode 100644 index a014623f9..000000000 --- a/internal/stackql/router/parameter_router.go +++ /dev/null @@ -1,751 +0,0 @@ -package router - -import ( - "fmt" - - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" - "github.com/stackql/stackql/internal/stackql/astindirect" - "github.com/stackql/stackql/internal/stackql/astvisit" - "github.com/stackql/stackql/internal/stackql/dataflow" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/router/obtain_context" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" -) - -var ( - _ ParameterRouter = &standardParameterRouter{} -) - -// Parameter router supports -// mapping columnar input to -// tabular output. -// This is for dealing with parser objects, prior to assignment -// of openapi schemas. -// The storage medium for constituents is abstracted. -// As of now this is a multi stage object, violates Single functionality. -type ParameterRouter interface { - - // Obtains parameters that are unammbiguous (eg: aliased, unique col name) - // or potential matches for a supplied table. - // getAvailableParameters(tb sqlparser.TableExpr) *parserutil.TableParameterCoupling - - // Records the fact that parameters have been assigned to a table and - // cannot be used elsewhere. - // splitParams(params map[string]interface{}) error - - // First pass assignment of columnar objects - // to tables, only for HTTP method parameters. All data accrual is done herein: - // - SQL parser table objects mapped to hierarchy. - // - Data flow dependencies identified and persisted. - // - Hierarchies may be persisted for analysis. - // Detects bi-directional data flow errors and returns error if so. - // Returns: - // - Hierarchy. - // - Columnar objects definitely assigned as HTTP method parameters. - // - Error if applicable. - Route(tb sqlparser.TableExpr, handler handler.HandlerContext, isAwait bool) (taxonomy.AnnotationCtx, error) - - // Detects: - // - Dependency cycle. - AnalyzeDependencies() error - - GetOnConditionsToRewrite() map[*sqlparser.ComparisonExpr]struct{} - - GetOnConditionDataFlows() (dataflow.Collection, error) -} - -type standardParameterRouter struct { - annotatedAST annotatedast.AnnotatedAst - tablesAliasMap parserutil.TableAliasMap - tableMap parserutil.TableExprMap - onParamMap parserutil.ParameterMap - whereParamMap parserutil.ParameterMap - colRefs parserutil.ColTableMap - comparisonToTableDependencies parserutil.ComparisonTableMap - tableToComparisonDependencies parserutil.ComparisonTableMap - tableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx - invalidatedParams map[string]interface{} - namespaceCollection tablenamespace.Collection - astFormatter sqlparser.NodeFormatter - dataFlowCfg dto.DataFlowCfg -} - -func NewParameterRouter( - annotatedAST annotatedast.AnnotatedAst, - tablesAliasMap parserutil.TableAliasMap, - tableMap parserutil.TableExprMap, - whereParamMap parserutil.ParameterMap, - onParamMap parserutil.ParameterMap, - colRefs parserutil.ColTableMap, - namespaceCollection tablenamespace.Collection, - astFormatter sqlparser.NodeFormatter, - dataflowCfg dto.DataFlowCfg, -) ParameterRouter { - return &standardParameterRouter{ - tablesAliasMap: tablesAliasMap, - tableMap: tableMap, - whereParamMap: whereParamMap, - onParamMap: onParamMap, - colRefs: colRefs, - invalidatedParams: make(map[string]interface{}), - comparisonToTableDependencies: make(parserutil.ComparisonTableMap), - tableToComparisonDependencies: make(parserutil.ComparisonTableMap), - tableToAnnotationCtx: make(map[sqlparser.TableExpr]taxonomy.AnnotationCtx), - namespaceCollection: namespaceCollection, - astFormatter: astFormatter, - annotatedAST: annotatedAST, - dataFlowCfg: dataflowCfg, - } -} - -func (pr *standardParameterRouter) AnalyzeDependencies() error { - return nil -} - -// This has been obviated for the time being. -// Essentially, not required so long as: -// - in params -// - result set data -// -// ...are guaranteed present in same table. -func (pr *standardParameterRouter) GetOnConditionsToRewrite() map[*sqlparser.ComparisonExpr]struct{} { - rv := make(map[*sqlparser.ComparisonExpr]struct{}) - for k := range pr.comparisonToTableDependencies { - logging.GetLogger().Debugf("%v\n", k) - } - return rv -} - -func (pr *standardParameterRouter) extractDataFlowDependency( - input sqlparser.Expr, -) (taxonomy.AnnotationCtx, sqlparser.TableExpr, error) { - switch l := input.(type) { - case *sqlparser.ColName: - // leave unknown for now -- bit of a mess - ref, err := parserutil.NewColumnarReference(l, parserutil.UnknownParam) - if err != nil { - return nil, nil, err - } - tb, ok := pr.colRefs[ref] - if !ok { - return nil, nil, fmt.Errorf("unassigned column in ON condition dataflow; please alias column '%s'", l.GetRawVal()) - } - hr, ok := pr.tableToAnnotationCtx[tb] - if !ok { - return nil, nil, fmt.Errorf("cannot assign hierarchy for column '%s'", l.GetRawVal()) - } - return hr, tb, nil - default: - return nil, nil, fmt.Errorf("cannot accomodate ON condition of type = '%T'", l) - } -} - -func (pr *standardParameterRouter) extractFromFunctionExpr( - f *sqlparser.FuncExpr, -) (taxonomy.AnnotationCtx, sqlparser.TableExpr, error) { - sv := astvisit.NewLeftoverReferencesAstVisitor( - pr.annotatedAST, - pr.colRefs, - pr.tableToAnnotationCtx, - ) - sv.Visit(f) //nolint:errcheck // TODO: review - tbz := sv.GetTablesFoundThisIteration() - if len(tbz) != 1 { - return nil, nil, fmt.Errorf("cannot accomodate this") - } - for k, v := range tbz { - return v, k, nil - } - return nil, nil, fmt.Errorf("cannot accomodate this") -} - -type paramSplitter interface { - split() (bool, error) - getSplitDataFlowCollection() dataflow.Collection - getSplitAnnotationContextMap() taxonomy.AnnotationCtxSplitMap -} - -type standardParamSplitter struct { - alreadySplit map[string]struct{} - splitParamMap map[sqlparser.TableExpr]map[string][]any - paramToTableExprMap map[string]sqlparser.TableExpr - splitAnnotationContextMap taxonomy.AnnotationCtxSplitMap - tableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx - dataflowCollection dataflow.Collection -} - -func newParamSplitter( - tableToAnnotationCtx map[sqlparser.TableExpr]taxonomy.AnnotationCtx, - dataFlowCfg dto.DataFlowCfg, -) paramSplitter { - splitParamMap := make(map[sqlparser.TableExpr]map[string][]any) - for k := range tableToAnnotationCtx { - splitParamMap[k] = make(map[string][]any) - } - return &standardParamSplitter{ - alreadySplit: make(map[string]struct{}), - splitParamMap: splitParamMap, - paramToTableExprMap: make(map[string]sqlparser.TableExpr), - splitAnnotationContextMap: taxonomy.NewAnnotationCtxSplitMap(), - tableToAnnotationCtx: tableToAnnotationCtx, - dataflowCollection: dataflow.NewStandardDataFlowCollection(dataFlowCfg), - } -} - -func (sp *standardParamSplitter) getSplitAnnotationContextMap() taxonomy.AnnotationCtxSplitMap { - return sp.splitAnnotationContextMap -} - -func (sp *standardParamSplitter) split() (bool, error) { - var tableEquivalencyID int64 - // Rist, see if any dependencies need splitting - var isAnythingSplit bool - for k, v := range sp.tableToAnnotationCtx { - tableEquivalencyID++ // start at 1 for > 0 logic - var skipBaseAdd bool - for k1, param := range v.GetParameters() { - sp.paramToTableExprMap[k1] = k - paramSlice, isSplit := sp.splitSingleParam(param) - sp.splitParamMap[k][k1] = paramSlice - if isSplit { - skipBaseAdd = true - isAnythingSplit = true - } - } - _, assembleErr := sp.assembleSplitParams(k, tableEquivalencyID) - if assembleErr != nil { - return false, assembleErr - } - if !skipBaseAdd { - sp.dataflowCollection.AddVertex(sp.dataflowCollection.UpsertStandardDataFlowVertex(v, k)) - } - } - return isAnythingSplit, nil -} - -func (sp *standardParamSplitter) getSplitDataFlowCollection() dataflow.Collection { - return sp.dataflowCollection -} - -func (sp *standardParamSplitter) assembleSplitParams( - tableExpr sqlparser.TableExpr, - tableEquivalencyID int64, -) (bool, error) { - rawAnnotationCtx, ok := sp.tableToAnnotationCtx[tableExpr] - if !ok { - return false, fmt.Errorf("table expression '%s' has not been assigned to hierarchy", sqlparser.String(tableExpr)) - } - combinationComposerObj := newCombinationComposer() - splitParams := sp.splitParamMap[tableExpr] - analysisErr := combinationComposerObj.analyse(splitParams) - if analysisErr != nil { - return false, analysisErr - } - combinations := combinationComposerObj.getCombinations() - _, isAnythingSplit := len(combinations), combinationComposerObj.getIsAnythingSplit() - for _, paramCombination := range combinations { - com := paramCombination - splitAnnotationCtx := taxonomy.NewStaticStandardAnnotationCtx( - rawAnnotationCtx.GetSchema(), - rawAnnotationCtx.GetHIDs(), - rawAnnotationCtx.GetTableMeta().Clone(), - com, - rawAnnotationCtx.IsAwait(), - ) - sp.splitAnnotationContextMap.Put(rawAnnotationCtx, splitAnnotationCtx) - // TODO: this has gotta replace the original and also be duplicated - sourceVertexIteration := sp.dataflowCollection.UpsertStandardDataFlowVertex(splitAnnotationCtx, tableExpr) - sourceVertexIteration.SetEquivalencyGroup(tableEquivalencyID) - sp.dataflowCollection.AddVertex(sourceVertexIteration) - } - return isAnythingSplit, nil -} - -func (sp *standardParamSplitter) splitSingleParam( - param any, -) ([]any, bool) { - var isSplit bool - var rv []any - switch param := param.(type) { //nolint:gocritic // TODO: review - case parserutil.ParameterMetadata: - rhs := param.GetVal() - switch rhs := rhs.(type) { - case sqlparser.ValTuple: - // TODO: fix update anomale for dataflow graph!!! - for _, valTmp := range rhs { - val := valTmp - rv = append(rv, val) - isSplit = true - } - default: - rv = append(rv, param) - } - } - return rv, isSplit -} - -//nolint:funlen,gocognit,nestif // inherently complex functionality -func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collection, error) { - paramSplitterObj := newParamSplitter(pr.tableToAnnotationCtx, pr.dataFlowCfg) - isInitiallySplit, splitErr := paramSplitterObj.split() - if isInitiallySplit { - logging.GetLogger().Debugf("dataflow required initial splitting") - } - if splitErr != nil { - return nil, splitErr - } - rv := paramSplitterObj.getSplitDataFlowCollection() - splitAnnotationContextMap := paramSplitterObj.getSplitAnnotationContextMap() - - for k, destinationTable := range pr.comparisonToTableDependencies { - selfTableCited := false - destHierarchy, ok := pr.tableToAnnotationCtx[destinationTable] - if !ok { - return nil, fmt.Errorf( - "table expression '%s' has not been assigned to hierarchy", sqlparser.String(destinationTable)) - } - var dependencyTable sqlparser.TableExpr - var dependencies []taxonomy.AnnotationCtx - var destinations []taxonomy.AnnotationCtx - - var destColumn *sqlparser.ColName - var srcExpr sqlparser.Expr - switch l := k.Left.(type) { - case *sqlparser.ColName: - lhr, candidateTable, err := pr.extractDataFlowDependency(l) - if err != nil { - return nil, err - } - if destHierarchy == lhr { - selfTableCited = true - destColumn = l - srcExpr = k.Right - } else { - splitDependencies, isSplit := splitAnnotationContextMap.Get(lhr) - if isSplit { - dependencies = append(dependencies, splitDependencies...) - } else { - dependencies = append(dependencies, lhr) - } - splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) - if isDestinationSplit { - destinations = append(destinations, splitDestinations...) - } else { - destinations = append(destinations, destHierarchy) - } - dependencyTable = candidateTable - srcExpr = k.Left - } - case *sqlparser.FuncExpr: - annCtx, te, err := pr.extractFromFunctionExpr(l) - if err != nil { - return nil, err - } - splitDependencies, isSplit := splitAnnotationContextMap.Get(annCtx) - if isSplit { - dependencies = append(dependencies, splitDependencies...) - } else { - dependencies = append(dependencies, annCtx) - } - splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) - if isDestinationSplit { - destinations = append(destinations, splitDestinations...) - } else { - destinations = append(destinations, destHierarchy) - } - dependencyTable = te - } - switch r := k.Right.(type) { - case *sqlparser.SQLVal: - // no dataflow dependencies, do zero - continue - case *sqlparser.ColName: - rhr, candidateTable, err := pr.extractDataFlowDependency(r) - if err != nil { - return nil, err - } - if destHierarchy == rhr { - if selfTableCited { - return nil, fmt.Errorf("table join ON comparison '%s' is self referencing", sqlparser.String(k)) - } - selfTableCited = true - destColumn = r - srcExpr = k.Left - } else { - splitDependencies, isSplit := splitAnnotationContextMap.Get(rhr) - if isSplit { - dependencies = append(dependencies, splitDependencies...) - } else { - dependencies = append(dependencies, rhr) - } - splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) - if isDestinationSplit { - destinations = append(destinations, splitDestinations...) - } else { - destinations = append(destinations, destHierarchy) - } - dependencyTable = candidateTable - } - case *sqlparser.FuncExpr: - annCtx, te, err := pr.extractFromFunctionExpr(r) - if err != nil { - return nil, err - } - splitDependencies, isSplit := splitAnnotationContextMap.Get(annCtx) - if isSplit { - dependencies = append(dependencies, splitDependencies...) - } else { - dependencies = append(dependencies, annCtx) - } - splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) - if isDestinationSplit { - destinations = append(destinations, splitDestinations...) - } else { - destinations = append(destinations, destHierarchy) - } - dependencyTable = te - } - if !selfTableCited { - return nil, fmt.Errorf("table join ON comparison '%s' referencing incomplete", sqlparser.String(k)) - } - - for i, dependency := range dependencies { - srcVertex := rv.UpsertStandardDataFlowVertex(dependency, dependencyTable) - destination := destHierarchy - if i < len(destinations) { - destination = destinations[i] - } - destVertex := rv.UpsertStandardDataFlowVertex(destination, destinationTable) - - err := rv.AddOrUpdateEdge( - srcVertex, - destVertex, - k, - srcExpr, - destColumn, - ) - if err != nil { - return nil, err - } - } - } - return rv.WithSplitAnnotationContextMap(splitAnnotationContextMap), nil -} - -// recoverNativeCasingParams re-keys snake_case parameters that the wire-name method matching -// dropped, back to their wire name via any-sdk's native-casing parameter set, so the request -// carries the value. consumed already holds the wire-matched params and is mutated in place; -// available is the pre-consumption set. It is a no-op for methods without request native -// casing (where nativeCasing equals the wire param set, so no supplied key is an alias). -func recoverNativeCasingParams( - consumed, available map[string]interface{}, - nativeCasing map[string]formulation.Addressable, -) { - for k, v := range available { - addr, isAlias := nativeCasing[k] - if !isAlias || addr.GetName() == k { - continue - } - if _, present := consumed[addr.GetName()]; !present { - consumed[addr.GetName()] = v - } - } -} - -//nolint:gocognit // who cares -func (pr *standardParameterRouter) getAvailableParameters( - tb sqlparser.TableExpr, -) parserutil.TableParameterCoupling { - rv := parserutil.NewTableParameterCoupling() - minKeyMap := make(map[string]int) - for k, v := range pr.whereParamMap.GetMap() { - key := k.String() - tableAlias := k.Alias() - ordinal := v.GetOrdinal() - foundTable, ok := pr.tablesAliasMap[tableAlias] - if ok && foundTable != tb { - continue - } - if pr.isInvalidated(key) { - continue - } - ref, ok := pr.colRefs[k] - if ok && ref != tb { - continue - } - existingOrdinal, ordinalOk := minKeyMap[key] - if ordinalOk { - _, isPlaceholder := v.(*parserutil.PlaceholderParameterMetadata) - if isPlaceholder { - continue - } - if existingOrdinal < ordinal { - continue - } - rv.DeleteByOrdinal(existingOrdinal) - } - rv.Add(k, v, parserutil.WhereParam) //nolint:errcheck // no issue - minKeyMap[key] = ordinal - } - for k, v := range pr.onParamMap.GetMap() { - key := k.String() - tableAlias := k.Alias() - foundTable, ok := pr.tablesAliasMap[tableAlias] - if ok && foundTable != tb { - continue - } - if pr.isInvalidated(key) { - continue - } - ref, ok := pr.colRefs[k] - if ok && ref != tb { - continue - } - val := v.GetVal() - switch val := val.(type) { //nolint:gocritic // TODO: review - case *sqlparser.ColName: - logging.GetLogger().Debugf("%v\n", val) - rhsAlias := val.Qualifier.GetRawVal() - logging.GetLogger().Debugf("%v\n", rhsAlias) - } - rv.Add(k, v, parserutil.JoinOnParam) //nolint:errcheck // no issue - } - return rv -} - -func (pr *standardParameterRouter) isInvalidated(key string) bool { - _, ok := pr.invalidatedParams[key] - return ok -} - -// Route will map columnar input to a supplied -// parser table object. -// Columnar input may come from either where clause -// or on conditions. -func (pr *standardParameterRouter) Route( - tb sqlparser.TableExpr, - handlerCtx handler.HandlerContext, - isAwait bool, -) (taxonomy.AnnotationCtx, error) { - return pr.route(tb, handlerCtx, isAwait) -} - -//nolint:funlen,gocognit,govet,gocyclo,cyclop // inherently complex functionality -func (pr *standardParameterRouter) route( - tb sqlparser.TableExpr, - handlerCtx handler.HandlerContext, - isAwait bool, -) (taxonomy.AnnotationCtx, error) { - // TODO: Get rid of the dead set mess that is where paramters in preference. - for k, v := range pr.whereParamMap.GetMap() { - logging.GetLogger().Infof("%v\n", v) - alias := k.Alias() - if alias == "" { - continue - } - t, ok := pr.tablesAliasMap[alias] - if !ok { - return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) - } - if t == tb { - ref, ok := pr.colRefs[k] - if ok && ref != t { - return nil, fmt.Errorf("failed parameter routing, cannot re-assign") - } - pr.colRefs[k] = t - } - } - for k, v := range pr.onParamMap.GetMap() { - logging.GetLogger().Infof("%v\n", v) - alias := k.Alias() - if alias == "" { - continue - } - t, ok := pr.tablesAliasMap[alias] - if !ok { - return nil, fmt.Errorf("alias '%s' does not map to any table expression", alias) - } - if t == tb { - ref, ok := pr.colRefs[k] - if ok && ref != t { - return nil, fmt.Errorf("failed parameter routing, cannot re-assign") - } - pr.colRefs[k] = t - } - } - // These are "available parameters" - tpc := pr.getAvailableParameters(tb) - runParamters := tpc.Clone() - // After executing GetHeirarchyFromStatement(), we know: - // - Any remaining param is not required. - // - Any "on" param that was consumed: - // - Can / must be from removed join conditions in a rewrite. [Requires Join in router for later rewrite]. - // - Defines a sequencing and data flow dependencies unless RHS is a literal. [Create new object to represent]. - // TODO: In order to do this, we can, for each table: - // 1. [*] Subtract the remaining parameters returned by GetHeirarchyFromStatement() - // from the available parameters. Will need reversible string to object translation. - // 2. [*] Identify "on" parameters that were consumed as per item #1. - // We are free to change the "table parameter coupling" API to accomodate - // items #1 and #2. - // 3. [*] If #2 is consumed, then: - // - [*] Tag the "on" comparison as being incident to the table. - // - [*] Tag the "on" comparison for later rewrite to NOP. - // Probably some - // new data structure to accomodate this. - // And then, once all tables are done and also therefore, all hierarchies are present: - // a) [ ] Assign all remaining on parameters based on schema. - // b) [ ] Represent assignments as edges from table to on condition. - // d) [ ] Throw error for disallowed scenarios: - // - Dual outgoing from ON object. - // e) [ ] Rewrite NOP on clauses. - // f) [ ] Catalogue and return dataflows (somehow) - // stringParams := tpc.GetStringified() - notOnParams := runParamters.Clone().GetNotOnCoupling() - priorNotOnParameters := notOnParams.Clone() - priorParameters := runParamters.Clone() - // notOnStringParams := notOnParams.GetStringified() - // TODO: add parent params into the mix here. - // Note: CTEs are converted to subqueries at AST level, so they follow - // the normal subquery path (handled via GetSubquery() in ObtainAnnotationCtx). - var hr tablemetadata.HeirarchyObjects - var err error - hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, notOnParams, false, isAwait) - if err != nil { - hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, runParamters, false, isAwait) - } else { - // If the where parameters are sufficient, then need to switch - // the Table - Paramater coupling object - runParamters = notOnParams - priorParameters = priorNotOnParameters - } - // logging.GetLogger().Infof("hr = '%+v', remainingParams = '%+v', err = '%+v'", hr, remainingParams, err) - if err != nil { - return nil, err - } - // reconstitutedConsumedParams, err := tpc.ReconstituteConsumedParams(remainingParams) - // if err != nil { - // return nil, err - // } - // reconstitutedConsumedParams := tpc.Minus(runParamters) - // reconstitutedConsumedParams := priorParameters.Minus(runParamters) - logging.GetLogger().Debugf("%v\n", priorParameters) - // TODO: need to get ALL the required stuff in here, - // BUT not send the wrong things for dataflow analysis. - reconstitutedConsumedParams := runParamters - abbreviatedConsumedMap, err := reconstitutedConsumedParams.AbbreviateMap() - if err != nil { - return nil, err - } - // Recover snake_case WHERE/INSERT keys that the wire-name method matching dropped: re-key - // them to their wire name via any-sdk's native-casing parameter set so the request carries - // the value. priorParameters holds the pre-consumption set; consumed (already wire-matched) - // params are untouched. No-op for methods without request native casing. - if hr != nil { - if method := hr.GetMethod(); method != nil { - if availableAbbrev, abbrevErr := priorParameters.AbbreviateMap(); abbrevErr == nil { - recoverNativeCasingParams( - abbreviatedConsumedMap, availableAbbrev, method.GetParametersIncludingNativeCasing()) - } - } - } - // if err != nil { - // return nil, err - // } - // TODO: fix this mess and make it global - // so that ancestor params can be correctly consumed!!! - // err = pr.splitParams(abbreviatedConsumedMap) - // if err != nil { - // return nil, err - // } - onConsumed := reconstitutedConsumedParams.GetOnCoupling() - pms := onConsumed.GetAllParameters() - logging.GetLogger().Infof("onConsumed = '%+v'", onConsumed) - for _, kv := range pms { - // In this stanza: - // 1. [*] mark comparisons for rewriting - // 2. [*] some sequencing data to be stored - p := kv.V.GetParent() - existingTable, ok := pr.comparisonToTableDependencies[p] - if ok { - return nil, fmt.Errorf( - "data flow violation detected: ON comparison expression '%s' is a dependencies for tables '%s' and '%s'", - sqlparser.String(p), sqlparser.String(existingTable), sqlparser.String(tb)) - } - pr.comparisonToTableDependencies[p] = tb - logging.GetLogger().Infof("%v", kv) - } - indirect, _ := pr.annotatedAST.GetIndirect(tb) - currentIndirect := indirect - // TODO: elide all non selected indirects - var alreadyMatched bool - for { - if currentIndirect == nil { - break - } - ind, matches := currentIndirect.MatchOnParams(abbreviatedConsumedMap) - if matches && !alreadyMatched { - indirect = ind - alreadyMatched = true - } else { - // elide this indirect - currentIndirect.SetElide(true) - } - var hasNext bool - currentIndirect, hasNext = currentIndirect.Next() - if !hasNext { - break - } - logging.GetLogger().Infof("nextIndirect = %v", currentIndirect) - } - hrView, hrViewPresent := hr.GetHeirarchyIDs().GetView() - if indirect == nil && hrViewPresent { //nolint:nestif // TODO: review - if hrView.IsMaterialized() { //nolint:gocritic // TODO: review - indirect, err = astindirect.NewMaterializedViewIndirect(hrView, handlerCtx.GetSQLSystem()) - if err != nil { - return nil, err - } - err = indirect.Parse() - if err != nil { - return nil, err - } - } else if hrView.IsTable() { - indirect, err = astindirect.NewPhysicalTableIndirect(hrView, handlerCtx.GetSQLSystem()) - if err != nil { - return nil, err - } - err = indirect.Parse() - if err != nil { - return nil, err - } - } else { - indirect, err = astindirect.NewViewIndirect(hrView) - if err != nil { - return nil, err - } - err = indirect.Parse() - if err != nil { - return nil, err - } - } - } - m := tablemetadata.NewExtendedTableMetadata( - hr, - taxonomy.GetTableNameFromStatement(tb, pr.astFormatter), - taxonomy.GetAliasFromStatement(tb)).WithIndirect(indirect) - - // store relationship from sqlparser table expression to - // hierarchy. This enables e2e relationship - // from expression to hierarchy. - // eg: "on" clause to openapi method - ac, err := obtain_context.ObtainAnnotationCtx( - handlerCtx.GetSQLSystem(), m, abbreviatedConsumedMap, pr.namespaceCollection) - pr.tableToAnnotationCtx[tb] = ac - return ac, err -} diff --git a/internal/stackql/router/table_routing.go b/internal/stackql/router/table_routing.go deleted file mode 100644 index deab9e9d9..000000000 --- a/internal/stackql/router/table_routing.go +++ /dev/null @@ -1,786 +0,0 @@ -//nolint:cyclop // inherently complex functionality -package router - -import ( - "fmt" - "strings" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/taxonomy" -) - -var ( - _ TableRouteAstVisitor = &standardTableRouteAstVisitor{} -) - -type TableRouteAstVisitor interface { - GetAnnotations() taxonomy.AnnotationCtxMap - GetParameterRouter() ParameterRouter - GetTableMap() taxonomy.TblMap - Visit(node sqlparser.SQLNode) error - IsPgInternalOnly() bool -} - -type standardTableRouteAstVisitor struct { - handlerCtx handler.HandlerContext - router ParameterRouter - tables taxonomy.TblMap - annotations taxonomy.AnnotationCtxMap - isAwait bool -} - -func NewTableRouteAstVisitor( - handlerCtx handler.HandlerContext, - router ParameterRouter, - isAwait bool, -) TableRouteAstVisitor { - return &standardTableRouteAstVisitor{ - handlerCtx: handlerCtx, - router: router, - tables: make(taxonomy.TblMap), - annotations: make(taxonomy.AnnotationCtxMap), - isAwait: isAwait, - } -} - -func (v *standardTableRouteAstVisitor) IsPgInternalOnly() bool { - for _, tbl := range v.tables { - if !tbl.IsPGInternalObject() { - return false - } - } - return len(v.tables) > 0 -} - -func (v *standardTableRouteAstVisitor) analyzeAliasedTable( - tb *sqlparser.AliasedTableExpr, -) (taxonomy.AnnotationCtx, error) { - switch ex := tb.Expr.(type) { - case *sqlparser.Subquery: - return v.router.Route(tb, v.handlerCtx, v.isAwait) - case sqlparser.TableName: - _, err := taxonomy.GetHeirarchyIDsFromParserNode(v.handlerCtx, ex) - if err != nil { - return nil, err - } - return v.router.Route(tb, v.handlerCtx, v.isAwait) - default: - return nil, fmt.Errorf("table of type '%T' not currently supported", ex) - } -} - -func (v *standardTableRouteAstVisitor) GetParameterRouter() ParameterRouter { - return v.router -} - -func (v *standardTableRouteAstVisitor) analyzeExecTableName( - tb *sqlparser.ExecSubquery, -) (taxonomy.AnnotationCtx, error) { - // cannot be recursive - return v.router.Route(tb, v.handlerCtx, v.isAwait) -} - -func (v *standardTableRouteAstVisitor) GetTableMap() taxonomy.TblMap { - return v.tables -} - -func (v *standardTableRouteAstVisitor) GetAnnotations() taxonomy.AnnotationCtxMap { - return v.annotations -} - -//nolint:funlen,gocognit,cyclop,gocyclo,staticcheck,gocritic,errcheck,govet,revive // inherently complex functionality -func (v *standardTableRouteAstVisitor) Visit(node sqlparser.SQLNode) error { - var err error - - switch node := node.(type) { - case *sqlparser.Select: - var options string - addIf := func(b bool, s string) { - if b { - options += s - } - } - addIf(node.Distinct, sqlparser.DistinctStr) - if node.Cache != nil { - if *node.Cache { - options += sqlparser.SQLCacheStr - } else { - options += sqlparser.SQLNoCacheStr - } - } - addIf(node.StraightJoinHint, sqlparser.StraightJoinHint) - addIf(node.SQLCalcFoundRows, sqlparser.SQLCalcFoundRowsStr) - - if node.Comments != nil { - node.Comments.Accept(v) //nolint:errcheck // TODO: review - } - if node.SelectExprs != nil { - node.SelectExprs.Accept(v) //nolint:errcheck // TODO: review - } - if node.From != nil { - fErr := node.From.Accept(v) - if fErr != nil { - return fErr - } - } - if node.Where != nil { - node.Where.Accept(v) //nolint:errcheck // TODO: review - } - if node.GroupBy != nil { - node.GroupBy.Accept(v) //nolint:errcheck // TODO: review - } - if node.Having != nil { - node.Having.Accept(v) //nolint:errcheck // TODO: review - } - if node.OrderBy != nil { - node.OrderBy.Accept(v) //nolint:errcheck // TODO: review - } - if node.Limit != nil { - node.Limit.Accept(v) //nolint:errcheck // TODO: review - } - return nil - - case *sqlparser.ParenSelect: - node.Accept(v) //nolint:errcheck // TODO: review - - case *sqlparser.Auth: - return nil - - case *sqlparser.AuthRevoke: - return nil - case *sqlparser.ExecSubquery: - annotation, aErr := v.analyzeExecTableName(node) - if aErr != nil { - return aErr - } - v.annotations[node] = annotation - t := annotation.GetTableMeta() - if t == nil { - return fmt.Errorf("nil table returned") - } - v.tables[node] = t - case *sqlparser.Exec: - return node.MethodName.Accept(v) - case *sqlparser.Sleep: - return nil - - case *sqlparser.Union: - err = node.FirstStatement.Accept(v) - if err != nil { - return err - } - for _, us := range node.UnionSelects { - err = us.Accept(v) - if err != nil { - return err - } - } - return nil - - case *sqlparser.UnionSelect: - return node.Statement.Accept(v) - - case *sqlparser.Stream: - err = node.Comments.Accept(v) - if err != nil { - return err - } - err = node.SelectExpr.Accept(v) - if err != nil { - return err - } - return node.Table.Accept(v) - - case *sqlparser.Insert: - - case *sqlparser.Update: - - case *sqlparser.Delete: - - case *sqlparser.Set: - - case *sqlparser.SetTransaction: - - case *sqlparser.DBDDL: - - case *sqlparser.DDL: - switch node.Action { - case sqlparser.CreateStr: - case sqlparser.DropStr: - case sqlparser.RenameStr: - case sqlparser.AlterStr: - case sqlparser.FlushStr: - case sqlparser.CreateVindexStr: - case sqlparser.DropVindexStr: - case sqlparser.AddVschemaTableStr: - case sqlparser.DropVschemaTableStr: - case sqlparser.AddColVindexStr: - case sqlparser.DropColVindexStr: - case sqlparser.AddSequenceStr: - case sqlparser.AddAutoIncStr: - default: - } - - case *sqlparser.OptLike: - - case *sqlparser.PartitionSpec: - switch node.Action { - case sqlparser.ReorganizeStr: - default: - } - - case *sqlparser.PartitionDefinition: - - case *sqlparser.TableSpec: - - case *sqlparser.ColumnDefinition: - - // Format returns a canonical string representation of the type and all relevant options - case *sqlparser.ColumnType: - ct := node - - if ct.Length != nil && ct.Scale != nil { - - } else if ct.Length != nil { - } - - if ct.EnumValues != nil { - } - - opts := make([]string, 0, 16) //nolint:mnd // TODO: review - if ct.Unsigned { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNSIGNED]) - } - if ct.Zerofill { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.ZEROFILL]) - } - if ct.Charset != "" { - opts = append( - opts, - sqlparser.KeywordStrings[sqlparser.CHARACTER], - sqlparser.KeywordStrings[sqlparser.SET], - ct.Charset) - } - if ct.Collate != "" { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COLLATE], ct.Collate) - } - if ct.NotNull { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.NOT], sqlparser.KeywordStrings[sqlparser.NULL]) - } - if ct.Default != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.DEFAULT], sqlparser.String(ct.Default)) - } - if ct.OnUpdate != nil { - opts = append( - opts, - sqlparser.KeywordStrings[sqlparser.ON], - sqlparser.KeywordStrings[sqlparser.UPDATE], - sqlparser.String(ct.OnUpdate)) - } - if ct.Autoincrement { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.AUTO_INCREMENT]) - } - if ct.Comment != nil { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.COMMENT_KEYWORD], sqlparser.String(ct.Comment)) - } - if ct.KeyOpt == sqlparser.ColKeyPrimary { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.PRIMARY], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeyUnique { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE]) - } - if ct.KeyOpt == sqlparser.ColKeyUniqueKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.UNIQUE], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKeySpatialKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.SPATIAL], sqlparser.KeywordStrings[sqlparser.KEY]) - } - if ct.KeyOpt == sqlparser.ColKey { - opts = append(opts, sqlparser.KeywordStrings[sqlparser.KEY]) - } - - if len(opts) != 0 { - } - - case *sqlparser.IndexDefinition: - idx := node - for i, col := range idx.Columns { - if i != 0 { - } else { - } - if col.Length != nil { - } - } - - for _, opt := range idx.Options { - if opt.Using != "" { - } else { - } - } - - case *sqlparser.IndexInfo: - ii := node - if ii.Primary { - } else { - if !ii.Name.IsEmpty() { - } - } - - case *sqlparser.AutoIncSpec: - - case *sqlparser.VindexSpec: - - numParams := len(node.Params) - if numParams != 0 { - for i, p := range node.Params { - logging.GetLogger().Debugf("%v\n", p) - if i != 0 { - } - } - } - - case sqlparser.VindexParam: - - case *sqlparser.ConstraintDefinition: - c := node - if c.Name != "" { - } - - case sqlparser.ReferenceAction: - a := node - switch a { //nolint:exhaustive // TODO: review - case sqlparser.Restrict: - case sqlparser.Cascade: - case sqlparser.NoAction: - case sqlparser.SetNull: - case sqlparser.SetDefault: - } - - case *sqlparser.ForeignKeyDefinition: - f := node - if f.OnDelete != sqlparser.DefaultAction { - } - if f.OnUpdate != sqlparser.DefaultAction { - } - - case *sqlparser.Show: - nodeType := strings.ToLower(node.Type) - //nolint:lll,nestif // TODO: review - if (nodeType == "tables" || nodeType == "columns" || nodeType == "fields" || nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.ShowTablesOpt != nil { - opt := node.ShowTablesOpt - if node.Extended != "" { - } else { - } - if (nodeType == "columns" || nodeType == "fields") && node.HasOnTable() { - } - if (nodeType == "index" || nodeType == "keys" || nodeType == "indexes") && node.HasOnTable() { - } - if opt.DbName != "" { - } - return nil - } - if node.Scope == "" { - } else { - } - if node.HasOnTable() { - } - if nodeType == "collation" && node.ShowCollationFilterOpt != nil { - } - if nodeType == "charset" && node.ShowTablesOpt != nil { - } - if node.HasTable() { - } - - case *sqlparser.ShowFilter: - if node == nil { - return nil - } - if node.Like != "" { - } else { - } - - case *sqlparser.Use: - if node.DBName.GetRawVal() != "" { - } else { - } - - case *sqlparser.Commit: - - case *sqlparser.Begin: - - case *sqlparser.Rollback: - - case *sqlparser.SRollback: - - case *sqlparser.Savepoint: - - case *sqlparser.Release: - - case *sqlparser.Explain: - switch node.Type { - case "": // do nothing - case sqlparser.AnalyzeStr: - default: - } - - case *sqlparser.OtherRead: - - case *sqlparser.DescribeTable: - - case *sqlparser.DescribeMethod: - - case *sqlparser.OtherAdmin: - - case sqlparser.Comments: - - case sqlparser.SelectExprs: - - case *sqlparser.StarExpr: - if !node.TableName.IsEmpty() { - } - - case *sqlparser.AliasedExpr: - if !node.As.IsEmpty() { - } - - case sqlparser.Nextval: - - case sqlparser.Columns: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.Partitions: - if node == nil { - return nil - } - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.TableExprs: - for _, n := range node { - if parserutil.IsNullTable(n) { - continue - } - err := n.Accept(v) - if err != nil { - return err - } - } - - case *sqlparser.AliasedTableExpr: - switch node.Expr.(type) { - case nil: - default: - annotation, err := v.analyzeAliasedTable(node) - if err != nil { - return err - } - v.annotations[node] = annotation - t := annotation.GetTableMeta() - if t == nil { - return fmt.Errorf("nil table returned") - } - v.tables[node] = t - } - if node.Partitions != nil { - node.Partitions.Accept(v) - } - if !node.As.IsEmpty() { - node.As.Accept(v) - } - if node.Hints != nil { - node.Hints.Accept(v) - } - - case sqlparser.TableNames: - for _, n := range node { - n.Accept(v) - } - - case sqlparser.TableName: - if node.IsEmpty() { - return nil - } - if !node.QualifierThird.IsEmpty() { - } - if !node.QualifierSecond.IsEmpty() { - } - if !node.Qualifier.IsEmpty() { - } - - case *sqlparser.ParenTableExpr: - - case sqlparser.JoinCondition: - if node.On != nil { - } - if node.Using != nil { - } - - case *sqlparser.JoinTableExpr: - err = node.LeftExpr.Accept(v) - if err != nil { - return err - } - err = node.RightExpr.Accept(v) - if err != nil { - return err - } - // If this is an OUTER JOIN - // then add some data to the annotation - // so that later passes can rewrite - // the ON clause with control - // attributes - // and also so that - // these are omitted from the WHERE clause. - switch node.Join { - case sqlparser.RightJoinStr: - case sqlparser.StraightJoinStr: - case sqlparser.JoinStr: - case sqlparser.LeftJoinStr, sqlparser.LeftOuterJoinStr: - // LEFT JOIN and LEFT OUTER JOIN are semantically identical in SQL. - rhs := v.tables[node.RightExpr] - if rhs == nil { - return fmt.Errorf("table routing: nil RHS table in left outer join") - } - rhs.SetIsOnClauseHoistable(true) - case sqlparser.RightOuterJoinStr: - lhs := v.tables[node.LeftExpr] - if lhs == nil { - return fmt.Errorf("table routing: nil LHS table in right outer join") - } - lhs.SetIsOnClauseHoistable(true) - case sqlparser.NaturalLeftJoinStr: - case sqlparser.NaturalRightJoinStr: - default: - return fmt.Errorf("table routing: unknown join type: %v", node.Join) - } - - case *sqlparser.IndexHints: - if len(node.Indexes) == 0 { - } else { - for _, n := range node.Indexes { - logging.GetLogger().Debugf("%v\n", n) - } - } - - case *sqlparser.Where: - if node == nil || node.Expr == nil { - return nil - } - return node.Expr.Accept(v) - - case sqlparser.Exprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.AndExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.OrExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.XorExpr: - err = node.Left.Accept(v) - if err != nil { - return err - } - return node.Right.Accept(v) - - case *sqlparser.NotExpr: - return node.Expr.Accept(v) - - case *sqlparser.ComparisonExpr: - return nil - - case *sqlparser.RangeCond: - - case *sqlparser.IsExpr: - return node.Expr.Accept(v) - - case *sqlparser.ExistsExpr: - return nil - - case *sqlparser.SQLVal: - switch node.Type { - case sqlparser.StrVal: - case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.HexNum: - case sqlparser.HexVal: - case sqlparser.BitVal: - case sqlparser.ValArg: - default: - } - - case *sqlparser.NullVal: - - case sqlparser.BoolVal: - if node { - } else { - } - - case *sqlparser.ColName: - if !node.Qualifier.IsEmpty() { - } - - case sqlparser.ValTuple: - - case *sqlparser.Subquery: - - case sqlparser.ListArg: - - case *sqlparser.BinaryExpr: - - case *sqlparser.UnaryExpr: - if _, unary := node.Expr.(*sqlparser.UnaryExpr); unary { - return nil - } - - case *sqlparser.IntervalExpr: - - case *sqlparser.TimestampFuncExpr: - - case *sqlparser.CurTimeFuncExpr: - - case *sqlparser.CollateExpr: - - case *sqlparser.FuncExpr: - if node.Distinct { - } - if !node.Qualifier.IsEmpty() { - } - // Function names should not be back-quoted even - // if they match a reserved word, only if they contain illegal characters - funcName := node.Name.String() - - if sqlparser.ContainEscapableChars(funcName, sqlparser.NoAt) { - } else { - } - - case *sqlparser.GroupConcatExpr: - - case *sqlparser.ValuesFuncExpr: - - case *sqlparser.SubstrExpr: - if node.Name != nil { - } else { - } - - if node.To == nil { - } else { - } - - case *sqlparser.ConvertExpr: - - case *sqlparser.ConvertUsingExpr: - - case *sqlparser.ConvertType: - if node.Length != nil { - if node.Scale != nil { - } - } - if node.Charset != "" { - } - - case *sqlparser.MatchExpr: - - case *sqlparser.CaseExpr: - if node.Expr != nil { - } - for _, when := range node.Whens { - logging.GetLogger().Debugf("%v\n", when) - } - if node.Else != nil { - } - - case *sqlparser.Default: - if node.ColName != "" { - } - - case *sqlparser.When: - - case sqlparser.GroupBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.OrderBy: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.Order: - if node, ok := node.Expr.(*sqlparser.NullVal); ok { - logging.GetLogger().Debugf("%v\n", node) - return nil - } - if node, ok := node.Expr.(*sqlparser.FuncExpr); ok { - if node.Name.Lowered() == "rand" { - return nil - } - } - - case *sqlparser.Limit: - if node == nil { - return nil - } - if node.Offset != nil { - } - - case sqlparser.Values: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case sqlparser.UpdateExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.UpdateExpr: - - case sqlparser.SetExprs: - for _, n := range node { - logging.GetLogger().Debugf("%v\n", n) - } - - case *sqlparser.SetExpr: - if node.Scope != "" { - } - // We don't have to backtick set variable names. - switch { - case node.Name.EqualString("charset") || node.Name.EqualString("names"): - case node.Name.EqualString(sqlparser.TransactionStr): - default: - } - - case sqlparser.OnDup: - if node == nil { - return nil - } - - case sqlparser.ColIdent: - for i := sqlparser.NoAt; i < node.GetAtCount(); i++ { - } - - case sqlparser.TableIdent: - - case *sqlparser.IsolationLevel: - - case *sqlparser.AccessMode: - } - return nil -} diff --git a/internal/stackql/runtimeconfig/auth_ctx.go b/internal/stackql/runtimeconfig/auth_ctx.go new file mode 100644 index 000000000..5ea2b6ce0 --- /dev/null +++ b/internal/stackql/runtimeconfig/auth_ctx.go @@ -0,0 +1,374 @@ +package dto + +import ( + "encoding/base64" + "fmt" + "net/url" + "os" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" +) + +type AuthContexts map[string]*AuthCtx + +func (as AuthContexts) Clone() AuthContexts { + rv := make(AuthContexts) + for k, v := range as { + rv[k] = v.Clone() + } + return rv +} + +type AuthCtx struct { + omnisdk.AuthConfig `yaml:",inline"` + SQLCfg *SQLBackendCfg `json:"sqlDataSource" yaml:"sqlDataSource"` + Successor *AuthCtx `json:"successor" yaml:"successor"` +} + +func (ac *AuthCtx) GetSQLCfg() (SQLBackendCfg, bool) { + var retVal SQLBackendCfg + if ac.SQLCfg != nil { + return *ac.SQLCfg, true + } + return retVal, false +} + +func (ac *AuthCtx) Clone() *AuthCtx { + if ac == nil { + return nil + } + cloned := *ac + cloned.Scopes = append([]string(nil), ac.Scopes...) + cloned.Values = make(url.Values, len(ac.Values)) + for key, values := range ac.Values { + cloned.Values[key] = append([]string(nil), values...) + } + cloned.Successor = ac.Successor.Clone() + if ac.SQLCfg != nil { + cfg := *ac.SQLCfg + cloned.SQLCfg = &cfg + } + return &cloned +} + +func (ac *AuthCtx) GetValues() url.Values { + if ac.Values == nil { + return url.Values{} + } + return ac.Values +} + +func (ac *AuthCtx) GetSuccessor() (*AuthCtx, bool) { + if ac.Successor != nil { + return ac.Successor, true + } + return nil, false +} + +func (ac *AuthCtx) GetInlineBasicCredentials() string { + if ac.Username != "" && ac.Password != "" { + plaintext := fmt.Sprintf("%s:%s", ac.Username, ac.Password) + encoded := base64.StdEncoding.EncodeToString([]byte(plaintext)) + return encoded + } + if ac.APIKeyStr != "" && ac.APISecretStr != "" { + plaintext := fmt.Sprintf("%s:%s", ac.APIKeyStr, ac.APISecretStr) + encoded := base64.StdEncoding.EncodeToString([]byte(plaintext)) + return encoded + } + return "" +} + +func (ac *AuthCtx) getEnvVarBasicCredentials() string { + if ac.EnvVarUsername != "" && ac.EnvVarPassword != "" { + userName := os.Getenv(ac.EnvVarUsername) + passWord := os.Getenv(ac.EnvVarPassword) + plaintext := fmt.Sprintf("%s:%s", userName, passWord) + encoded := base64.StdEncoding.EncodeToString([]byte(plaintext)) + return encoded + } + if ac.EnvVarAPIKeyStr != "" && ac.EnvVarAPISecretStr != "" { + userName := os.Getenv(ac.EnvVarAPIKeyStr) + passWord := os.Getenv(ac.EnvVarAPISecretStr) + plaintext := fmt.Sprintf("%s:%s", userName, passWord) + encoded := base64.StdEncoding.EncodeToString([]byte(plaintext)) + return encoded + } + return "" +} + +func (ac *AuthCtx) HasKey() bool { + if ac.KeyFilePath != "" || ac.KeyEnvVar != "" { + return true + } + return false +} + +func (ac *AuthCtx) GetKeyIDString() (string, error) { + if ac.KeyIDEnvVar != "" { + rv := os.Getenv(ac.KeyIDEnvVar) + if rv == "" { + return "", fmt.Errorf("keyIDenvvar references empty string") + } + return rv, nil + } + return ac.KeyID, nil +} + +func (ac *AuthCtx) GetAwsSessionTokenString() (string, error) { + token := os.Getenv("AWS_SESSION_TOKEN") + return token, nil // Session token is optional, so an empty token isn't considered an error. +} + +// GetAwsRoleArn resolves the ARN of the role to assume, preferring the +// environment variable indirection when supplied. The role ARN is mandatory +// for the aws_assume_role auth type. +func (ac *AuthCtx) GetAwsRoleArn() (string, error) { + if ac.AwsRoleArnEnvVar != "" { + rv := os.Getenv(ac.AwsRoleArnEnvVar) + if rv == "" { + return "", fmt.Errorf("aws_role_arn_env_var references empty string") + } + return rv, nil + } + if ac.AwsRoleArn == "" { + return "", fmt.Errorf("aws_role_arn is empty") + } + return ac.AwsRoleArn, nil +} + +// GetAwsRoleSessionName returns the configured STS session name, falling back to +// a deterministic default when none is supplied. AWS requires a session name on +// every AssumeRole call. +func (ac *AuthCtx) GetAwsRoleSessionName() string { + if ac.AwsRoleSessionName != "" { + return ac.AwsRoleSessionName + } + return "stackql-assume-role-session" +} + +// GetAwsRoleExternalID resolves the optional STS external ID, preferring the +// environment variable indirection when supplied. An empty result is valid. +func (ac *AuthCtx) GetAwsRoleExternalID() string { + if ac.AwsRoleExternalIDEnvVar != "" { + return os.Getenv(ac.AwsRoleExternalIDEnvVar) + } + return ac.AwsRoleExternalID +} + +// GetAwsStsRegion returns the region used to reach the STS endpoint when +// assuming a role, defaulting to us-east-1. This is independent of the region +// used to sign the eventual service request. +func (ac *AuthCtx) GetAwsStsRegion() string { + if ac.AwsStsRegion != "" { + return ac.AwsStsRegion + } + return "us-east-1" +} + +const ( + defaultOciConfigFilePath string = "~/.oci/config" + defaultOciProfile string = "DEFAULT" +) + +// GetOciTenancyOCID resolves the OCI tenancy OCID, preferring the environment +// variable indirection when supplied. An empty result means "not configured" +// and triggers config-file fallback. +func (ac *AuthCtx) GetOciTenancyOCID() string { + if ac.OciTenancyOCIDEnvVar != "" { + return os.Getenv(ac.OciTenancyOCIDEnvVar) + } + return ac.OciTenancyOCID +} + +// GetOciUserOCID resolves the OCI user OCID, preferring the environment +// variable indirection when supplied. +func (ac *AuthCtx) GetOciUserOCID() string { + if ac.OciUserOCIDEnvVar != "" { + return os.Getenv(ac.OciUserOCIDEnvVar) + } + return ac.OciUserOCID +} + +// GetOciFingerprint resolves the fingerprint of the API signing key, preferring +// the environment variable indirection when supplied. +func (ac *AuthCtx) GetOciFingerprint() string { + if ac.OciFingerprintEnvVar != "" { + return os.Getenv(ac.OciFingerprintEnvVar) + } + return ac.OciFingerprint +} + +// GetOciPrivateKey resolves the RSA private key PEM: an env var carrying the +// PEM itself wins, then an inline literal, then a key file path (literal or via +// env var). An empty result with nil error means "not configured" and triggers +// config-file fallback. +func (ac *AuthCtx) GetOciPrivateKey() (string, error) { + if ac.OciPrivateKeyEnvVar != "" { + rv := os.Getenv(ac.OciPrivateKeyEnvVar) + if rv == "" { + return "", fmt.Errorf("private_key_env_var references empty string") + } + return rv, nil + } + if ac.OciPrivateKey != "" { + return ac.OciPrivateKey, nil + } + keyFilePath := ac.OciPrivateKeyFilePath + if ac.OciPrivateKeyFilePathEnvVar != "" { + keyFilePath = os.Getenv(ac.OciPrivateKeyFilePathEnvVar) + if keyFilePath == "" { + return "", fmt.Errorf("private_key_path_env_var references empty string") + } + } + if keyFilePath == "" { + return "", nil + } + b, err := os.ReadFile(keyFilePath) // #nosec G703 -- credential file is explicitly configured by the user. + if err != nil { + return "", err + } + return string(b), nil +} + +// GetOciPassphrase resolves the optional private key passphrase, preferring the +// environment variable indirection when supplied. An empty result is valid. +func (ac *AuthCtx) GetOciPassphrase() string { + if ac.OciPassphraseEnvVar != "" { + return os.Getenv(ac.OciPassphraseEnvVar) + } + return ac.OciPassphrase +} + +// GetOciRegion resolves the region, preferring the environment variable +// indirection when supplied. The region does not participate in request +// signing; it is a provider-config convenience. +func (ac *AuthCtx) GetOciRegion() string { + if ac.OciRegionEnvVar != "" { + return os.Getenv(ac.OciRegionEnvVar) + } + return ac.OciRegion +} + +// GetOciConfigFilePath returns the OCI config file path, defaulting to the +// ~/.oci/config convention every OCI tool shares. +func (ac *AuthCtx) GetOciConfigFilePath() string { + if ac.OciConfigFilePath != "" { + return ac.OciConfigFilePath + } + return defaultOciConfigFilePath +} + +// GetOciProfile returns the OCI config file profile, defaulting to DEFAULT. +func (ac *AuthCtx) GetOciProfile() string { + if ac.OciProfile != "" { + return ac.OciProfile + } + return defaultOciProfile +} + +func (ac *AuthCtx) InferAuthType(authTypeRequested string) string { + ft := strings.ToLower(authTypeRequested) + switch ft { + case AuthAPIKeyStr: + return AuthAPIKeyStr + case AuthServiceAccountStr: + return AuthServiceAccountStr + case AuthInteractiveStr: + return AuthInteractiveStr + } + if ac.KeyFilePath != "" || ac.KeyEnvVar != "" || ac.KeyFilePathEnvVar != "" { + return AuthServiceAccountStr + } + return AuthInteractiveStr +} + +func (ac *AuthCtx) GetCredentialsBytes() ([]byte, error) { + if ac.KeyEnvVar != "" { + rv := os.Getenv(ac.KeyEnvVar) + if rv == "" { + return nil, fmt.Errorf("credentialsenvvar references empty string") + } + return []byte(rv), nil + } + if ac.KeyFilePathEnvVar != "" { + credentialFile := os.Getenv(ac.KeyFilePathEnvVar) + return os.ReadFile(credentialFile) // #nosec G703 -- credential file is explicitly configured by the user. + } + credentialFile := ac.KeyFilePath + if credentialFile != "" { + return os.ReadFile(credentialFile) // #nosec G703 -- credential file is explicitly configured by the user. + } + if ac.getEnvVarBasicCredentials() != "" { + return []byte(ac.getEnvVarBasicCredentials()), nil + } + if ac.GetInlineBasicCredentials() != "" { + return []byte(ac.GetInlineBasicCredentials()), nil + } + if ac.EncodedBasicCredentials != "" { + return []byte(ac.EncodedBasicCredentials), nil + } + return nil, fmt.Errorf("no credentials found") +} + +func (ac *AuthCtx) GetClientID() (string, error) { + if ac.ClientIDEnvVar != "" { + rv := os.Getenv(ac.ClientIDEnvVar) + if rv == "" { + return "", fmt.Errorf("client_id_env_var references empty string") + } + return rv, nil + } + if ac.ClientID == "" { + return "", fmt.Errorf("client_id is empty") + } + return ac.ClientID, nil +} + +func (ac *AuthCtx) GetClientSecret() (string, error) { + if ac.ClientSecretEnvVar != "" { + rv := os.Getenv(ac.ClientSecretEnvVar) + if rv == "" { + return "", fmt.Errorf("client_secret_env_var references empty string") + } + return rv, nil + } + if ac.ClientSecret == "" { + return "", fmt.Errorf("client_secret is empty") + } + return ac.ClientSecret, nil +} + +func (ac *AuthCtx) GetGrantType() string { + return ac.GrantType +} + +func (ac *AuthCtx) GetTokenURL() string { + return ac.TokenURL +} + +func (ac *AuthCtx) GetAuthStyle() int { + return ac.AuthStyle +} + +func (ac *AuthCtx) GetCredentialsSourceDescriptorString() string { + if ac.KeyEnvVar != "" { + return fmt.Sprintf("credentialsenvvar:%s", ac.KeyEnvVar) + } + return fmt.Sprintf("credentialsfilepath:%s", ac.KeyFilePath) +} + +func GetAuthCtx(scopes []string, keyFilePath string, keyFileType string) *AuthCtx { + var authType string + if keyFilePath == "" { + authType = AuthInteractiveStr + } else { + authType = inferKeyFileType(keyFileType) + } + return &AuthCtx{AuthConfig: omnisdk.AuthConfig{Scopes: scopes, + Type: authType, + KeyFilePath: keyFilePath, + Active: false}, + } +} diff --git a/internal/stackql/runtimeconfig/context_keys.go b/internal/stackql/runtimeconfig/context_keys.go new file mode 100644 index 000000000..51315ca46 --- /dev/null +++ b/internal/stackql/runtimeconfig/context_keys.go @@ -0,0 +1,23 @@ +package dto + +import "context" + +type ContextKey string + +const ( + ContextPrefixStackqlRequest string = "stackql/request/" +) + +const ( + ContextKeyCreationDate ContextKey = "stackql/request/created_date" +) + +func ExtractStackqlRequestContextValue[T any](ctx context.Context, key ContextKey) (T, bool) { + val := ctx.Value(key) + if val == nil { + var zero T + return zero, false + } + typedVal, ok := val.(T) + return typedVal, ok +} diff --git a/internal/stackql/runtimeconfig/dataflow_cfg.go b/internal/stackql/runtimeconfig/dataflow_cfg.go new file mode 100644 index 000000000..3e947938e --- /dev/null +++ b/internal/stackql/runtimeconfig/dataflow_cfg.go @@ -0,0 +1,30 @@ +package dto + +var ( + _ DataFlowCfg = &dataFlowCfg{} +) + +type DataFlowCfg interface { + GetMaxDependencies() int + GetMaxComponents() int +} + +type dataFlowCfg struct { + maxDependencies int + maxComponents int +} + +func NewDataFlowCfg(maxDependencies int, maxComponents int) DataFlowCfg { + return &dataFlowCfg{ + maxDependencies: maxDependencies, + maxComponents: maxComponents, + } +} + +func (d *dataFlowCfg) GetMaxDependencies() int { + return d.maxDependencies +} + +func (d *dataFlowCfg) GetMaxComponents() int { + return d.maxComponents +} diff --git a/internal/stackql/runtimeconfig/dto.go b/internal/stackql/runtimeconfig/dto.go new file mode 100644 index 000000000..ad518649f --- /dev/null +++ b/internal/stackql/runtimeconfig/dto.go @@ -0,0 +1,80 @@ +package dto + +const ( + AuthAPIKeyStr string = "api_key" + AuthAWSSigningv4Str string = "aws_signing_v4" + AuthAWSAssumeRoleStr string = "aws_assume_role" + AuthAzureDefaultStr string = "azure_default" + AuthOciSigningv1Str string = "oci_signing_v1" + AuthBasicStr string = "basic" + AuthBearerStr string = "bearer" + AuthCustomStr string = "custom" + AuthInteractiveStr string = "interactive" + AuthServiceAccountStr string = "service_account" + AuthNullStr string = "null_auth" + DryRunFlagKey string = "dryrun" + ExecutionConcurrencyLimitKey string = "execution.concurrency.limit" + AuthCtxKey string = "auth" + APIRequestTimeoutKey string = "apirequesttimeout" + CacheKeyCountKey string = "cachekeycount" + CacheTTLKey string = "metadatattl" + ClientCredentialsStr string = "client_credentials" + ColorSchemeKey string = "colorscheme" // deprecated + ConfigFilePathKey string = "configfile" + CPUProfileKey string = "cpuprofile" + CSVHeadersDisableKey string = "hideheaders" + DelimiterKey string = "delimiter" + ErrorPresentationKey string = "errorpresentation" + IndirectDepthMaxKey string = "indirect.depth.max" + DataflowDependencyMaxKey string = "dataflow.dependency.max" + DataflowComponentsMaxKey string = "dataflow.components.max" + HTTPLogEnabledKey string = "http.log.enabled" + HTTPMaxResultsKey string = "http.response.maxResults" + HTTPPAgeLimitKey string = "http.response.pageLimit" + HTTPProxyHostKey string = "http.proxy.host" + HTTPProxyPasswordKey string = "http.proxy.password" //nolint:gosec // no hardcoded credentials + HTTPProxyPortKey string = "http.proxy.port" + HTTPProxySchemeKey string = "http.proxy.scheme" + HTTPProxyUserKey string = "http.proxy.user" + CABundleKey string = "tls.CABundle" + AllowInsecureKey string = "tls.allowInsecure" + InfilePathKey string = "infile" + LogLevelStrKey string = "loglevel" + OAuth2Str string = "oauth2" + OutfilePathKey string = "outfile" + OutputFormatKey string = "output" + ApplicationFilesRootPathKey string = "approot" + ApplicationFilesRootPathModeKey string = "approotfilemode" + PgSrvAddressKey string = "pgsrv.address" + ExportAliasKey string = "export.alias" + PgSrvLogLevelKey string = "pgsrv.loglevel" + PgSrvPortKey string = "pgsrv.port" + PgSrvRawTLSCfgKey string = "pgsrv.tls" + PgSrvRawSrvCfgKey string = "pgsrv.sundry" + PgSrvIsDebugNoticesEnabledKey string = "pgsrv.debug.enable" + ProviderStrKey string = "provider" + QueryCacheSizeKey string = "querycachesize" + RegistryRawKey string = "registry" + SessionCtxKey string = "session" + GCCfgRawKey string = "gc" + UpstreamErrorCfgRawKey string = "upstream.errors" + ACIDCfgRawKey string = "acid" + NamespaceCfgRawKey string = "namespaces" + SQLBackendCfgRawKey string = "sqlBackend" + DBInternalCfgRawKey string = "dbInternal" + StoreTxnCfgRawKey string = "store.txn" + TemplateCtxFilePathKey string = "iqldata" + TestWithoutAPICallsKey string = "TestWithoutAPICalls" + UseNonPreferredAPIsKEy string = "usenonpreferredapis" + VarListKey string = "var" + VerboseFlagKey string = "verbose" + ViperCfgFileNameKey string = "viperconfigfilename" + WorkOfflineKey string = "offline" +) + +func inferKeyFileType(keyFileType string) string { + if keyFileType == "" { + return AuthServiceAccountStr + } + return keyFileType +} diff --git a/internal/stackql/runtimeconfig/gc_config.go b/internal/stackql/runtimeconfig/gc_config.go new file mode 100644 index 000000000..fbf21258c --- /dev/null +++ b/internal/stackql/runtimeconfig/gc_config.go @@ -0,0 +1,15 @@ +package dto + +import ( + "gopkg.in/yaml.v2" +) + +type GCCfg struct { + IsEager bool `json:"isEager" yaml:"isEager"` +} + +func GetGCCfg(s string) (GCCfg, error) { + rv := GCCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} diff --git a/internal/stackql/runtimeconfig/kstore_config.go b/internal/stackql/runtimeconfig/kstore_config.go new file mode 100644 index 000000000..37293cc36 --- /dev/null +++ b/internal/stackql/runtimeconfig/kstore_config.go @@ -0,0 +1,15 @@ +package dto + +import ( + "gopkg.in/yaml.v2" +) + +type KStoreCfg struct { + IsPlaceholder bool `json:"isPlaceholder" yaml:"isPlaceholder"` +} + +func GetKStoreCfg(s string) (KStoreCfg, error) { + rv := KStoreCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} diff --git a/internal/stackql/runtimeconfig/marshalled_body.go b/internal/stackql/runtimeconfig/marshalled_body.go new file mode 100644 index 000000000..249557971 --- /dev/null +++ b/internal/stackql/runtimeconfig/marshalled_body.go @@ -0,0 +1,26 @@ +package dto + +type MarshalledBody interface { + GetBytes() []byte + GetError() (error, bool) +} + +type standardMarshalledBody struct { + bytes []byte + err error +} + +func NewMarshalledBody(b []byte, e error) MarshalledBody { + return &standardMarshalledBody{ + bytes: b, + err: e, + } +} + +func (mb *standardMarshalledBody) GetBytes() []byte { + return mb.bytes +} + +func (mb *standardMarshalledBody) GetError() (error, bool) { + return mb.err, mb.err != nil +} diff --git a/internal/stackql/runtimeconfig/namespace_config.go b/internal/stackql/runtimeconfig/namespace_config.go new file mode 100644 index 000000000..c862ce884 --- /dev/null +++ b/internal/stackql/runtimeconfig/namespace_config.go @@ -0,0 +1,32 @@ +package dto + +import ( + "regexp" + "text/template" + + "gopkg.in/yaml.v2" +) + +type NamespaceCfg struct { + RegexpStr string `json:"regex" yaml:"regex"` + TTL int `json:"ttl" yaml:"ttl"` + NamespaceTemplate string `json:"template" yaml:"template"` +} + +func (nc NamespaceCfg) GetRegex() (*regexp.Regexp, error) { + return regexp.Compile(nc.RegexpStr) +} + +func (nc NamespaceCfg) GetTemplate() (*template.Template, error) { + tmpl, err := template.New("stackqlNamespaceTmpl").Parse(nc.NamespaceTemplate) + if err != nil { + return nil, err + } + return tmpl, nil +} + +func GetNamespaceCfg(s string) (map[string]NamespaceCfg, error) { + rv := make(map[string]NamespaceCfg) + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} diff --git a/internal/stackql/runtimeconfig/omnisdk_auth.go b/internal/stackql/runtimeconfig/omnisdk_auth.go new file mode 100644 index 000000000..8ea76c6d3 --- /dev/null +++ b/internal/stackql/runtimeconfig/omnisdk_auth.go @@ -0,0 +1,66 @@ +package dto + +import ( + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" +) + +func OmniAuth(authCtx *AuthCtx) *omnisdk.Auth { + if authCtx == nil { + return nil + } + auth := &omnisdk.Auth{ + TenancyOCID: authCtx.OciTenancyOCID, + TenancyOCIDEnvVar: authCtx.OciTenancyOCIDEnvVar, + UserOCID: authCtx.OciUserOCID, + UserOCIDEnvVar: authCtx.OciUserOCIDEnvVar, + Fingerprint: authCtx.OciFingerprint, + FingerprintEnvVar: authCtx.OciFingerprintEnvVar, + PrivateKey: authCtx.OciPrivateKey, + PrivateKeyEnvVar: authCtx.OciPrivateKeyEnvVar, + PrivateKeyPath: authCtx.OciPrivateKeyFilePath, + PrivateKeyPathEnvVar: authCtx.OciPrivateKeyFilePathEnvVar, + Passphrase: authCtx.OciPassphrase, + PassphraseEnvVar: authCtx.OciPassphraseEnvVar, + ConfigFilePath: authCtx.OciConfigFilePath, + Profile: authCtx.OciProfile, + + Type: authCtx.Type, + ValuePrefix: authCtx.ValuePrefix, + Name: authCtx.Name, + Scopes: authCtx.Scopes, + TokenURL: authCtx.GetTokenURL(), + Username: authCtx.Username, + Password: authCtx.Password, + UsernameEnvVar: authCtx.EnvVarUsername, + PasswordEnvVar: authCtx.EnvVarPassword, + } + if strings.EqualFold(authCtx.Type, "basic") && auth.ValuePrefix == "" { + // Preserve the CLI's established Basic scheme spelling. + auth.ValuePrefix = "BASIC " + } + if (strings.EqualFold(authCtx.Type, "api_key") || strings.EqualFold(authCtx.Type, "custom")) && auth.Name == "" { + auth.Name = "Authorization" + if auth.ValuePrefix == "" && strings.EqualFold(authCtx.Type, "api_key") { + auth.ValuePrefix = "Bearer " + } + } + if credentials, credErr := authCtx.GetCredentialsBytes(); credErr == nil { + auth.SecretAccessKey = string(credentials) + auth.Credentials = string(credentials) + } + if keyID, keyErr := authCtx.GetKeyIDString(); keyErr == nil { + auth.AccessKeyID = keyID + } + if sessionToken, tokErr := authCtx.GetAwsSessionTokenString(); tokErr == nil { + auth.SessionToken = sessionToken + } + if clientID, idErr := authCtx.GetClientID(); idErr == nil { + auth.ClientID = clientID + } + if clientSecret, secErr := authCtx.GetClientSecret(); secErr == nil { + auth.ClientSecret = clientSecret + } + return auth +} diff --git a/internal/stackql/runtimeconfig/output_packet.go b/internal/stackql/runtimeconfig/output_packet.go new file mode 100644 index 000000000..2c8f78ca2 --- /dev/null +++ b/internal/stackql/runtimeconfig/output_packet.go @@ -0,0 +1,57 @@ +package dto + +import ( + "github.com/lib/pq/oid" +) + +type OutputPacket interface { + GetRows() map[string]map[string]interface{} + GetRawRows() map[int]map[int]interface{} + GetColumnNames() []string + GetColumnOIDs() []oid.Oid + GetMessages() []string +} + +func NewStandardOutputPacket( + rowMaps map[string]map[string]interface{}, + rawRows map[int]map[int]interface{}, + columnNames []string, + columnOIDs []oid.Oid, + messages []string, +) OutputPacket { + return &standardOutputPacket{ + rowMaps: rowMaps, + rawRows: rawRows, + columnNames: columnNames, + columnOIDs: columnOIDs, + messages: messages, + } +} + +type standardOutputPacket struct { + rowMaps map[string]map[string]interface{} + rawRows map[int]map[int]interface{} + columnNames []string + columnOIDs []oid.Oid + messages []string +} + +func (op *standardOutputPacket) GetRows() map[string]map[string]interface{} { + return op.rowMaps +} + +func (op *standardOutputPacket) GetMessages() []string { + return op.messages +} + +func (op *standardOutputPacket) GetRawRows() map[int]map[int]interface{} { + return op.rawRows +} + +func (op *standardOutputPacket) GetColumnNames() []string { + return op.columnNames +} + +func (op *standardOutputPacket) GetColumnOIDs() []oid.Oid { + return op.columnOIDs +} diff --git a/internal/stackql/runtimeconfig/pg_internal_config.go b/internal/stackql/runtimeconfig/pg_internal_config.go new file mode 100644 index 000000000..96502b4f5 --- /dev/null +++ b/internal/stackql/runtimeconfig/pg_internal_config.go @@ -0,0 +1,18 @@ +package dto + +import ( + "gopkg.in/yaml.v2" +) + +type DBMSInternalCfg struct { + ShowRegex string `json:"showRegex" yaml:"showRegex"` + TableRegex string `json:"tableRegex" yaml:"tableRegex"` + SchemaRegex string `json:"schemaRegex" yaml:"schemaRegex"` + FuncRegex string `json:"funcRegex" yaml:"funcRegex"` +} + +func GetDBMSInternalCfg(s string) (DBMSInternalCfg, error) { + rv := DBMSInternalCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} diff --git a/internal/stackql/runtimeconfig/pg_tls_dto.go b/internal/stackql/runtimeconfig/pg_tls_dto.go new file mode 100644 index 000000000..8a2270e94 --- /dev/null +++ b/internal/stackql/runtimeconfig/pg_tls_dto.go @@ -0,0 +1,18 @@ +package dto + +import "crypto/tls" + +type PgTLSCfg struct { + KeyFilePath string `json:"keyFilePath" yaml:"keyFilePath"` + CertFilePath string `json:"certFilePath" yaml:"certFilePath"` + KeyContents string `json:"keyContents" yaml:"keyContents"` + CertContents string `json:"certContents" yaml:"certContents"` + ClientCAs []string `json:"clientCAs" yaml:"clientCAs"` +} + +func (pc PgTLSCfg) GetKeyPair() (tls.Certificate, error) { + if len(pc.KeyContents) > 0 && len(pc.CertContents) > 0 { + return tls.X509KeyPair([]byte(pc.CertContents), []byte(pc.KeyContents)) + } + return tls.LoadX509KeyPair(pc.CertFilePath, pc.KeyFilePath) +} diff --git a/internal/stackql/runtimeconfig/runtime_ctx.go b/internal/stackql/runtimeconfig/runtime_ctx.go new file mode 100644 index 000000000..62cf117db --- /dev/null +++ b/internal/stackql/runtimeconfig/runtime_ctx.go @@ -0,0 +1,312 @@ +package dto + +import "strconv" + +type RuntimeCtx struct { + APIRequestTimeout int + AuthRaw string + CABundle string + AllowInsecure bool + CacheKeyCount int + CacheTTL int + ConfigFilePath string + CPUProfile string + CSVHeadersDisable bool + Delimiter string + DryRunFlag bool + ErrorPresentation string + ExecutionConcurrencyLimit int + HTTPLogEnabled bool + HTTPMaxResults int + HTTPPageLimit int + HTTPProxyHost string + HTTPProxyPassword string + HTTPProxyPort int + HTTPProxyScheme string + HTTPProxyUser string + IndirectDepthMax int + DataflowComponentsMax int + DataflowDependencyMax int + InfilePath string + LogLevelStr string + OutfilePath string + OutputFormat string + ApplicationFilesRootPath string + ApplicationFilesRootPathMode uint32 + PGSrvAddress string + PGSrvLogLevel string + PGSrvPort int + PGSrvRawTLSCfg string + PGSrvRawSrvCfg string + PGSrvIsDebugNoticesEnabled bool + ExportAlias string + ProviderStr string + RegistryRaw string + SessionCtxRaw string + SQLBackendCfgRaw string + DBInternalCfgRaw string + NamespaceCfgRaw string + StoreTxnCfgRaw string + GCCfgRaw string + ACIDCfgRaw string + UpstreamErrorCfgRaw string + StrictUpstreamErrors bool + QueryCacheSize int + TemplateCtxFilePath string + TestWithoutAPICalls bool + UseNonPreferredAPIs bool + VarList []string + VerboseFlag bool + ViperCfgFileName string + WorkOffline bool + CLIPayload string + CLIPayloadType string + CLIParameters string + CLIProvFilePath string + CLISvcFilePath string + CLIProviderName string + CLIResourceStr string + CLIMethodName string + CLISchemaDir string + CLISkipSchemaValidation bool + CLIRewriteURL string + CLIMockOutputDir string + CLIMockExpectationDir string + CLIMockQueryDir string + CLIProviderOut string + CLIClosureOutputDir string + CLIStdoutFile string + CLIStderrFile string +} + +func setInt(iPtr *int, val string) error { + i, err := strconv.Atoi(val) + if err == nil { + *iPtr = i + } + return err +} + +func setUint32(uPtr *uint32, val string) error { + ui, err := strconv.ParseUint(val, 10, 32) + if err == nil { + *uPtr = uint32(ui) + } + return err +} + +func setBool(bPtr *bool, val string) error { + b, err := strconv.ParseBool(val) + if err == nil { + *bPtr = b + } + return err +} + +//nolint:funlen,gocyclo,cyclop // pretty much boileplate +func (rc *RuntimeCtx) Set(key string, val string) error { + var retVal error + switch key { + case APIRequestTimeoutKey: + retVal = setInt(&rc.APIRequestTimeout, val) + case AuthCtxKey: + rc.AuthRaw = val + case CABundleKey: + rc.CABundle = val + case CacheKeyCountKey: + retVal = setInt(&rc.CacheKeyCount, val) + case CacheTTLKey: + retVal = setInt(&rc.CacheTTL, val) + case ColorSchemeKey: // deprecated + case ConfigFilePathKey: + rc.ConfigFilePath = val + case CPUProfileKey: + rc.CPUProfile = val + case CSVHeadersDisableKey: + retVal = setBool(&rc.CSVHeadersDisable, val) + case SQLBackendCfgRawKey: + rc.SQLBackendCfgRaw = val + case DBInternalCfgRawKey: + rc.DBInternalCfgRaw = val + case DelimiterKey: + rc.Delimiter = val + case DryRunFlagKey: + retVal = setBool(&rc.DryRunFlag, val) + case ErrorPresentationKey: + rc.ErrorPresentation = val + case ExecutionConcurrencyLimitKey: + retVal = setInt(&rc.ExecutionConcurrencyLimit, val) + case HTTPLogEnabledKey: + retVal = setBool(&rc.HTTPLogEnabled, val) + case HTTPMaxResultsKey: + retVal = setInt(&rc.HTTPMaxResults, val) + case HTTPPAgeLimitKey: + retVal = setInt(&rc.HTTPPageLimit, val) + case HTTPProxyHostKey: + rc.HTTPProxyHost = val + case HTTPProxyPasswordKey: + rc.HTTPProxyPassword = val + case HTTPProxyPortKey: + retVal = setInt(&rc.HTTPProxyPort, val) + case IndirectDepthMaxKey: + retVal = setInt(&rc.IndirectDepthMax, val) + case DataflowComponentsMaxKey: + retVal = setInt(&rc.DataflowComponentsMax, val) + case DataflowDependencyMaxKey: + retVal = setInt(&rc.DataflowDependencyMax, val) + case HTTPProxySchemeKey: + rc.HTTPProxyScheme = val + case HTTPProxyUserKey: + rc.HTTPProxyUser = val + case InfilePathKey: + rc.InfilePath = val + case LogLevelStrKey: + rc.LogLevelStr = val + case NamespaceCfgRawKey: + rc.NamespaceCfgRaw = val + case StoreTxnCfgRawKey: + rc.StoreTxnCfgRaw = val + case GCCfgRawKey: + rc.GCCfgRaw = val + case ACIDCfgRawKey: + rc.ACIDCfgRaw = val + case UpstreamErrorCfgRawKey: + rc.UpstreamErrorCfgRaw = val + case OutfilePathKey: + rc.OutfilePath = val + case OutputFormatKey: + rc.OutputFormat = val + case ApplicationFilesRootPathKey: + rc.ApplicationFilesRootPath = val + case ApplicationFilesRootPathModeKey: + retVal = setUint32(&rc.ApplicationFilesRootPathMode, val) + case PgSrvAddressKey: + rc.PGSrvAddress = val + case PgSrvLogLevelKey: + rc.PGSrvLogLevel = val + case ExportAliasKey: + rc.ExportAlias = val + case PgSrvPortKey: + retVal = setInt(&rc.PGSrvPort, val) + case PgSrvRawTLSCfgKey: + rc.PGSrvRawTLSCfg = val + case PgSrvRawSrvCfgKey: + rc.PGSrvRawSrvCfg = val + case PgSrvIsDebugNoticesEnabledKey: + retVal = setBool(&rc.PGSrvIsDebugNoticesEnabled, val) + case QueryCacheSizeKey: + retVal = setInt(&rc.QueryCacheSize, val) + case RegistryRawKey: + rc.RegistryRaw = val + case SessionCtxKey: + rc.SessionCtxRaw = val + case TemplateCtxFilePathKey: + rc.TemplateCtxFilePath = val + case TestWithoutAPICallsKey: + retVal = setBool(&rc.TestWithoutAPICalls, val) + case UseNonPreferredAPIsKEy: + retVal = setBool(&rc.UseNonPreferredAPIs, val) + case VarListKey: + rc.VarList = append(rc.VarList, val) + case VerboseFlagKey: + retVal = setBool(&rc.VerboseFlag, val) + case ViperCfgFileNameKey: + rc.ViperCfgFileName = val + case WorkOfflineKey: + retVal = setBool(&rc.WorkOffline, val) + } + return retVal +} + +func (rc RuntimeCtx) Copy() RuntimeCtx { + return RuntimeCtx{ + APIRequestTimeout: rc.APIRequestTimeout, + AuthRaw: rc.AuthRaw, + CABundle: rc.CABundle, + AllowInsecure: rc.AllowInsecure, + CacheKeyCount: rc.CacheKeyCount, + CacheTTL: rc.CacheTTL, + ConfigFilePath: rc.ConfigFilePath, + CPUProfile: rc.CPUProfile, + CSVHeadersDisable: rc.CSVHeadersDisable, + Delimiter: rc.Delimiter, + DryRunFlag: rc.DryRunFlag, + ErrorPresentation: rc.ErrorPresentation, + ExecutionConcurrencyLimit: rc.ExecutionConcurrencyLimit, + HTTPLogEnabled: rc.HTTPLogEnabled, + HTTPMaxResults: rc.HTTPMaxResults, + HTTPPageLimit: rc.HTTPPageLimit, + HTTPProxyHost: rc.HTTPProxyHost, + HTTPProxyPassword: rc.HTTPProxyPassword, + HTTPProxyPort: rc.HTTPProxyPort, + HTTPProxyScheme: rc.HTTPProxyScheme, + HTTPProxyUser: rc.HTTPProxyUser, + IndirectDepthMax: rc.IndirectDepthMax, + DataflowComponentsMax: rc.DataflowComponentsMax, + DataflowDependencyMax: rc.DataflowDependencyMax, + InfilePath: rc.InfilePath, + LogLevelStr: rc.LogLevelStr, + OutfilePath: rc.OutfilePath, + OutputFormat: rc.OutputFormat, + ApplicationFilesRootPath: rc.ApplicationFilesRootPath, + ApplicationFilesRootPathMode: rc.ApplicationFilesRootPathMode, + PGSrvAddress: rc.PGSrvAddress, + PGSrvLogLevel: rc.PGSrvLogLevel, + PGSrvPort: rc.PGSrvPort, + PGSrvRawTLSCfg: rc.PGSrvRawTLSCfg, + PGSrvRawSrvCfg: rc.PGSrvRawSrvCfg, + PGSrvIsDebugNoticesEnabled: rc.PGSrvIsDebugNoticesEnabled, + ExportAlias: rc.ExportAlias, + ProviderStr: rc.ProviderStr, + RegistryRaw: rc.RegistryRaw, + SessionCtxRaw: rc.SessionCtxRaw, + SQLBackendCfgRaw: rc.SQLBackendCfgRaw, + DBInternalCfgRaw: rc.DBInternalCfgRaw, + NamespaceCfgRaw: rc.NamespaceCfgRaw, + StoreTxnCfgRaw: rc.StoreTxnCfgRaw, + GCCfgRaw: rc.GCCfgRaw, + ACIDCfgRaw: rc.ACIDCfgRaw, + UpstreamErrorCfgRaw: rc.UpstreamErrorCfgRaw, + StrictUpstreamErrors: rc.StrictUpstreamErrors, + QueryCacheSize: rc.QueryCacheSize, + TemplateCtxFilePath: rc.TemplateCtxFilePath, + TestWithoutAPICalls: rc.TestWithoutAPICalls, + UseNonPreferredAPIs: rc.UseNonPreferredAPIs, + VarList: rc.VarList, + VerboseFlag: rc.VerboseFlag, + ViperCfgFileName: rc.ViperCfgFileName, + WorkOffline: rc.WorkOffline, + } +} + +func (rc RuntimeCtx) GetCABundle() string { + return rc.CABundle +} + +func (rc RuntimeCtx) GetHTTPProxyHost() string { + return rc.HTTPProxyHost +} + +func (rc RuntimeCtx) GetHTTPProxyUser() string { + return rc.HTTPProxyUser +} + +func (rc RuntimeCtx) GetHTTPProxyPassword() string { + return rc.HTTPProxyPassword +} + +func (rc RuntimeCtx) GetHTTPProxyPort() int { + return rc.HTTPProxyPort +} + +func (rc RuntimeCtx) GetHTTPProxyScheme() string { + return rc.HTTPProxyScheme +} + +func (rc RuntimeCtx) GetAPIRequestTimeout() int { + return rc.APIRequestTimeout +} + +func (rc RuntimeCtx) GetTLSAllowInsecure() bool { + return rc.AllowInsecure +} diff --git a/internal/stackql/runtimeconfig/session_context.go b/internal/stackql/runtimeconfig/session_context.go new file mode 100644 index 000000000..1cca9f02d --- /dev/null +++ b/internal/stackql/runtimeconfig/session_context.go @@ -0,0 +1,122 @@ +package dto + +import ( + "fmt" + + "github.com/stackql/stackql/internal/stackql/constants" + + "gopkg.in/yaml.v3" +) + +var ( + _ SessionContext = &sessionContext{} +) + +type SessionCtxConfig struct { + IsolationLevel string `json:"isolation_level" yaml:"isolation_level"` + RollbackType string `json:"rollback_type" yaml:"rollback_type"` +} + +type SessionContext interface { + Clone() SessionContext + GetIsolationLevel() constants.IsolationLevel + UpdateIsolationLevel(string) error + GetRollbackType() constants.RollbackType + UpdateRollbackType(string) error +} + +type sessionContext struct { + isolationLevel constants.IsolationLevel + rollbackType constants.RollbackType +} + +func NewSessionContext(cfgStr string) (SessionContext, error) { + var cfg SessionCtxConfig + err := yaml.Unmarshal([]byte(cfgStr), &cfg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal session context config: %w", err) + } + isolationLevelStr := cfg.IsolationLevel + if isolationLevelStr == "" { + isolationLevelStr = constants.ReadUncommittedStr + } + rollbackTypeStr := cfg.RollbackType + if rollbackTypeStr == "" { + rollbackTypeStr = constants.NopRollbackStr + } + isolationLevel, isolationLevelErr := inferIsolationLevel(isolationLevelStr) + if isolationLevelErr != nil { + return nil, fmt.Errorf("failed to infer isolation level: %w", isolationLevelErr) + } + rollbackType, rollbackTypeErr := inferRollbackType(rollbackTypeStr) + if rollbackTypeErr != nil { + return nil, fmt.Errorf("failed to infer rollback type: %w", rollbackTypeErr) + } + return &sessionContext{ + isolationLevel: isolationLevel, + rollbackType: rollbackType, + }, nil +} + +func (sc *sessionContext) GetIsolationLevel() constants.IsolationLevel { + return sc.isolationLevel +} + +func inferIsolationLevel(isolationLevelStr string) (constants.IsolationLevel, error) { + isolationLevel := constants.ReadUncommitted + switch isolationLevelStr { + case constants.ReadUncommittedStr: + isolationLevel = constants.ReadUncommitted + case constants.ReadCommittedStr: + isolationLevel = constants.ReadCommitted + case constants.RepeatableReadStr: + isolationLevel = constants.RepeatableRead + case constants.SerializableStr: + isolationLevel = constants.Serializable + default: + return isolationLevel, fmt.Errorf("invalid isolation level: %s", isolationLevelStr) + } + return isolationLevel, nil +} + +func inferRollbackType(rollbackTypeStr string) (constants.RollbackType, error) { + rollbackType := constants.NopRollback + switch rollbackTypeStr { + case constants.NopRollbackStr: + rollbackType = constants.NopRollback + case constants.EagerRollbackStr: + rollbackType = constants.EagerRollback + default: + return rollbackType, fmt.Errorf("invalid rollback type: %s", rollbackTypeStr) + } + return rollbackType, nil +} + +func (sc *sessionContext) UpdateIsolationLevel(isolationLevelStr string) error { + isolationLevel, isolationLevelErr := inferIsolationLevel(isolationLevelStr) + if isolationLevelErr != nil { + return fmt.Errorf("failed to infer isolation level: %w", isolationLevelErr) + } + sc.isolationLevel = isolationLevel + return nil +} + +func (sc *sessionContext) GetRollbackType() constants.RollbackType { + return sc.rollbackType +} + +func (sc *sessionContext) UpdateRollbackType(rollbackTypeStr string) error { + rollbackType, rollbackTypeErr := inferRollbackType(rollbackTypeStr) + if rollbackTypeErr != nil { + return fmt.Errorf("failed to infer rollback type: %w", rollbackTypeErr) + } + sc.rollbackType = rollbackType + return nil +} + +func (sc *sessionContext) Clone() SessionContext { + return &sessionContext{ + isolationLevel: sc.isolationLevel, + rollbackType: sc.rollbackType, + } +} diff --git a/internal/stackql/runtimeconfig/sql_backend_config.go b/internal/stackql/runtimeconfig/sql_backend_config.go new file mode 100644 index 000000000..a9f449de5 --- /dev/null +++ b/internal/stackql/runtimeconfig/sql_backend_config.go @@ -0,0 +1,90 @@ +package dto + +import ( + "fmt" + "os" + "strings" + + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/xo/dburl" + "gopkg.in/yaml.v2" +) + +type SQLBackendSchemata struct { + TableSchema string `json:"tableSchema" yaml:"tableSchema"` + IntelViewSchema string `json:"intelViewSchema" yaml:"intelViewSchema"` + OpsViewSchema string `json:"opsViewSchema" yaml:"opsViewSchema"` +} + +type SQLBackendCfg struct { + DBEngine string `json:"dbEngine" yaml:"dbEngine"` + DSN string `json:"dsn" yaml:"dsn"` + DSNEnvVar string `json:"dsnEnvVar" yaml:"dsnEnvVar"` + Schemata SQLBackendSchemata `json:"schemata" yaml:"schemata"` + DbInitFilePath string `json:"dbInitFilepath" yaml:"dbInitFilepath"` //nolint:stylecheck,lll // parity with JSON key + SQLSystem string `json:"sqlDialect" yaml:"sqlDialect"` + SchemaType string `json:"schemaType" yaml:"schemaType"` + InitMaxRetries int `json:"initMaxRetries" yaml:"initMaxRetries"` + InitRetryInitialDelay int `json:"initRetryInitialDelay" yaml:"initRetryInitialDelay"` +} + +func (sqlCfg SQLBackendCfg) GetDSN() string { + dsn := sqlCfg.DSN + if sqlCfg.DSNEnvVar != "" { + dsn = os.Getenv(sqlCfg.DSNEnvVar) + } + return dsn +} + +func (sqlCfg SQLBackendCfg) GetSchemaType() string { + return sqlCfg.SchemaType +} + +func (sqlCfg SQLBackendCfg) GetSQLDialect() string { + return sqlCfg.SQLSystem +} + +func (sqlCfg SQLBackendCfg) GetDatabaseName() (string, error) { + dsn := sqlCfg.GetDSN() + if dsn == "" { + return "", fmt.Errorf("cannot ionfer db name from empty dsn") + } + dbURL, err := dburl.Parse(dsn) + if err != nil { + return "", fmt.Errorf("error parsing postgres dsn: %w", err) + } + if dbURL == nil { + return "", fmt.Errorf("error parsing postgres dsn, nil url generated") + } + return strings.TrimLeft(dbURL.Path, "/"), nil +} + +func (sqlCfg SQLBackendCfg) GetTableSchemaName() string { + return sqlCfg.Schemata.TableSchema +} + +func (sqlCfg SQLBackendCfg) GetOpsViewSchemaName() string { + return sqlCfg.Schemata.OpsViewSchema +} + +func (sqlCfg SQLBackendCfg) GetIntelViewSchemaName() string { + return sqlCfg.Schemata.IntelViewSchema +} + +func GetSQLBackendCfg(s string) (SQLBackendCfg, error) { + rv := SQLBackendCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + if rv.DBEngine == "" { + rv.DBEngine = constants.DBEngineDefault + } + if rv.SQLSystem == "" { + rv.SQLSystem = constants.SQLDialectDefault + } + if rv.InitMaxRetries < 1 { + rv.InitMaxRetries = 10 + } + if rv.InitRetryInitialDelay < 1 { + rv.InitRetryInitialDelay = 10 + } + return rv, err +} diff --git a/internal/stackql/runtimeconfig/txn_coordinator_config.go b/internal/stackql/runtimeconfig/txn_coordinator_config.go new file mode 100644 index 000000000..8827c9e69 --- /dev/null +++ b/internal/stackql/runtimeconfig/txn_coordinator_config.go @@ -0,0 +1,26 @@ +package dto + +import ( + "gopkg.in/yaml.v2" +) + +const ( + defaultMaxTxnDepth = -1 +) + +type TxnCoordinatorCfg struct { + MaxTxnDepth *int `json:"maxTransactionDepth" yaml:"maxTransactionDepth"` +} + +func (t TxnCoordinatorCfg) GetMaxTxnDepth() int { + if t.MaxTxnDepth == nil { + return defaultMaxTxnDepth + } + return *t.MaxTxnDepth +} + +func GetTxnCoordinatorCfgCfg(s string) (TxnCoordinatorCfg, error) { + rv := TxnCoordinatorCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} diff --git a/internal/stackql/runtimeconfig/upstream_error_config.go b/internal/stackql/runtimeconfig/upstream_error_config.go new file mode 100644 index 000000000..db4c1e0b4 --- /dev/null +++ b/internal/stackql/runtimeconfig/upstream_error_config.go @@ -0,0 +1,41 @@ +package dto + +import ( + "strconv" + "strings" + + "gopkg.in/yaml.v2" +) + +// UpstreamErrorCfg is the opt in policy for upstream errors that a read +// tolerates, keyed by protocol. The zero value tolerates nothing, which is +// the default behaviour. +type UpstreamErrorCfg struct { + HTTP HTTPUpstreamErrorCfg `json:"http" yaml:"http"` +} + +// HTTPUpstreamErrorCfg lists the HTTP response status codes that a read +// ignores, either exactly (eg "503") or by class (eg "5xx"). +type HTTPUpstreamErrorCfg struct { + IgnoreStatusCodes []string `json:"ignoreStatusCodes" yaml:"ignoreStatusCodes"` +} + +func GetUpstreamErrorCfg(s string) (UpstreamErrorCfg, error) { + rv := UpstreamErrorCfg{} + err := yaml.Unmarshal([]byte(s), &rv) + return rv, err +} + +func (c HTTPUpstreamErrorCfg) IsIgnored(statusCode int) bool { + code := strconv.Itoa(statusCode) + for _, pattern := range c.IgnoreStatusCodes { + pattern = strings.ToLower(strings.TrimSpace(pattern)) + if pattern == code { + return true + } + if len(pattern) == len(code) && strings.HasSuffix(pattern, "xx") && pattern[0] == code[0] { + return true + } + } + return false +} diff --git a/internal/stackql/sql_system/cache_freshness_internal_test.go b/internal/stackql/sql_system/cache_freshness_internal_test.go index 7c7815b80..def46b257 100644 --- a/internal/stackql/sql_system/cache_freshness_internal_test.go +++ b/internal/stackql/sql_system/cache_freshness_internal_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" ) // queryOnlyEngine satisfies sqlengine.SQLEngine for the one method the cache diff --git a/internal/stackql/sql_system/postgres.go b/internal/stackql/sql_system/postgres.go index e9ee6a8f0..063101e6a 100644 --- a/internal/stackql/sql_system/postgres.go +++ b/internal/stackql/sql_system/postgres.go @@ -10,16 +10,16 @@ import ( "time" "github.com/lib/pq/oid" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astfuncrewrite" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto" + "github.com/stackql/stackql/internal/stackql/logging" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/pkg/serde" "github.com/stackql/stackql/pkg/textutil" @@ -144,14 +144,14 @@ func (eng *postgresSystem) GenerateDDL( func (eng *postgresSystem) RegisterExternalTable( connectionName string, - tableDetails formulation.SQLExternalTable, + tableDetails sqlengine.SQLExternalTable, ) error { return eng.registerExternalTable(connectionName, tableDetails) } func (eng *postgresSystem) registerExternalTable( connectionName string, - tableDetails formulation.SQLExternalTable, + tableDetails sqlengine.SQLExternalTable, ) error { q := ` INSERT INTO "__iql__.external.columns" ( @@ -640,10 +640,7 @@ func (eng *postgresSystem) RefreshMaterializedView(naiveViewName string, varargs ...any) error { fullyQualifiedRelationName := eng.getFullyQualifiedRelationName(naiveViewName) //nolint:gosec // no viable alternative - deleteQuery := fmt.Sprintf(` - DELETE FROM "%s"`, - fullyQualifiedRelationName, - ) + deleteQuery := "DELETE FROM " + eng.DelimitFullyQualifiedRelationName(fullyQualifiedRelationName) txn, err := eng.sqlEngine.GetTx() if err != nil { return err @@ -661,7 +658,8 @@ func (eng *postgresSystem) RefreshMaterializedView(naiveViewName string, // no need to rollbak; assumed already done return fmt.Errorf("cannot refresh materialized view = '%s': not found", naiveViewName) } - insertQuery := eng.generateTableInsertDMLFromViewSelect(fullyQualifiedRelationName, selectQuery, colz) + insertQuery := eng.generateTableInsertDMLFromViewSelect( + eng.DelimitFullyQualifiedRelationName(fullyQualifiedRelationName), selectQuery, colz) _, err = txn.Exec(insertQuery, varargs...) if err != nil { txn.Rollback() @@ -985,7 +983,11 @@ func (eng *postgresSystem) GetFullyQualifiedRelationName(tableName string) strin } func (eng *postgresSystem) DelimitFullyQualifiedRelationName(fqtn string) string { - return fqtn + quote := func(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + if eng.IsRelationExported(fqtn) { + return quote(eng.exportNamespace) + "." + quote(strings.TrimPrefix(fqtn, eng.exportNamespace+".")) + } + return quote(fqtn) } func (eng *postgresSystem) getFullyQualifiedRelationName(tableName string) string { @@ -1883,13 +1885,14 @@ func (eng *postgresSystem) runMaterializedViewCreate( return err } } - tableDDL := eng.generateTableDDL(naiveRelationName, colz) + tableDDL := eng.generateTableDDL(eng.DelimitFullyQualifiedRelationName(naiveRelationName), colz) _, err := txn.Exec(tableDDL) if err != nil { txn.Rollback() return err } - insertQuery := eng.generateTableInsertDMLFromViewSelect(naiveRelationName, selectQuery, colz) + insertQuery := eng.generateTableInsertDMLFromViewSelect( + eng.DelimitFullyQualifiedRelationName(naiveRelationName), selectQuery, colz) _, err = txn.Exec(insertQuery, varargs...) if err != nil { txn.Rollback() diff --git a/internal/stackql/sql_system/setup_scripts.go b/internal/stackql/sql_system/setup_scripts.go index 28c650e14..16fc28c80 100644 --- a/internal/stackql/sql_system/setup_scripts.go +++ b/internal/stackql/sql_system/setup_scripts.go @@ -1,7 +1,7 @@ package sql_system //nolint:stylecheck,revive // package name is meaningful and readable import ( - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/sqlengine" ) //nolint:gochecknoglobals // fine with this diff --git a/internal/stackql/sql_system/sql_system.go b/internal/stackql/sql_system/sql_system.go index 1d1ec832a..ab567fee4 100644 --- a/internal/stackql/sql_system/sql_system.go +++ b/internal/stackql/sql_system/sql_system.go @@ -7,15 +7,15 @@ import ( "strings" "time" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astfuncrewrite" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/pkg/astformat" ) @@ -110,7 +110,7 @@ type SQLSystem interface { varargs ...any) error // External SQL data sources - RegisterExternalTable(connectionName string, tableDetails formulation.SQLExternalTable) error + RegisterExternalTable(connectionName string, tableDetails sqlengine.SQLExternalTable) error ObtainRelationalColumnFromExternalSQLtable( hierarchyIDs internaldto.HeirarchyIdentifiers, colName string, diff --git a/internal/stackql/sql_system/sqlite.go b/internal/stackql/sql_system/sqlite.go index 1aaaf257c..152e1fade 100644 --- a/internal/stackql/sql_system/sqlite.go +++ b/internal/stackql/sql_system/sqlite.go @@ -10,16 +10,16 @@ import ( "time" "github.com/lib/pq/oid" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astfuncrewrite" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto" + "github.com/stackql/stackql/internal/stackql/logging" + "github.com/stackql/stackql/internal/stackql/sqlengine" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" "github.com/stackql/stackql/internal/stackql/typing" "github.com/stackql/stackql/pkg/serde" @@ -206,14 +206,14 @@ func (eng *sqLiteSystem) GCCollectObsoleted(minTransactionID int) error { func (eng *sqLiteSystem) RegisterExternalTable( connectionName string, - tableDetails formulation.SQLExternalTable, + tableDetails sqlengine.SQLExternalTable, ) error { return eng.registerExternalTable(connectionName, tableDetails) } func (eng *sqLiteSystem) registerExternalTable( connectionName string, - tableDetails formulation.SQLExternalTable, + tableDetails sqlengine.SQLExternalTable, ) error { q := ` INSERT OR IGNORE INTO "__iql__.external.columns" ( diff --git a/internal/stackql/sqlengine/config.go b/internal/stackql/sqlengine/config.go new file mode 100644 index 000000000..5141c54c6 --- /dev/null +++ b/internal/stackql/sqlengine/config.go @@ -0,0 +1,11 @@ +package sqlengine + +// Config is the host's resolved database connection configuration. Environment +// variables are resolved by the caller before constructing it. +type Config struct { + DBEngine string `json:"dbEngine"` + DSN string `json:"dsn"` + DbInitFilePath string `json:"dbInitFilepath"` + InitMaxRetries int `json:"initMaxRetries"` + InitRetryInitialDelay int `json:"initRetryInitialDelay"` +} diff --git a/internal/stackql/sqlengine/connect/db_util.go b/internal/stackql/sqlengine/connect/db_util.go new file mode 100644 index 000000000..38969db29 --- /dev/null +++ b/internal/stackql/sqlengine/connect/db_util.go @@ -0,0 +1,49 @@ +package db_util //nolint:revive,stylecheck // decent package name + +import ( + "database/sql" + "fmt" + "time" + + "github.com/stackql/stackql/internal/stackql/logging" +) + +func Open(driverName, dbName, dsn string, maxRetries, retryDelaySeconds int) (*sql.DB, error) { + if dsn == "" { + return nil, fmt.Errorf("cannot init %s TCP connection with empty dsn", dbName) + } + db, err := sql.Open(driverName, dsn) + retryCount := 0 + for { + if retryCount >= maxRetries || err == nil { + break + } + time.Sleep(time.Duration(retryDelaySeconds) * time.Second) + db, err = sql.Open(driverName, dsn) + retryCount++ + } + if err != nil { + return nil, fmt.Errorf("%s db object setup error = '%w'", driverName, err) + } + logging.GetLogger().Debugln( + fmt.Sprintf("opened %s TCP db with connection string = '%s' and err = '%v'", + dbName, dsn, err)) + pingErr := db.Ping() + retryCount = 0 + for { + if retryCount >= maxRetries || pingErr == nil { + break + } + time.Sleep(time.Duration(retryDelaySeconds) * time.Second) + pingErr = db.Ping() + retryCount++ + } + if pingErr != nil { + return nil, fmt.Errorf("%s connection setup ping error = '%w'", dbName, pingErr) + } + logging.GetLogger().Debugln( + fmt.Sprintf( + "opened and pinged %s TCP db with connection string = '%s' and err = '%v'", + dbName, dsn, err)) + return db, nil +} diff --git a/internal/stackql/sqlengine/control/sqlcontrol.go b/internal/stackql/sqlengine/control/sqlcontrol.go new file mode 100644 index 000000000..c53b29277 --- /dev/null +++ b/internal/stackql/sqlengine/control/sqlcontrol.go @@ -0,0 +1,82 @@ +package sqlcontrol + +import ( + "strings" +) + +var ( + _ ControlAttributes = &standardControlAttributes{} +) + +const ( + genIDColName string = "iql_generation_id" + ssnIDColName string = "iql_session_id" + txnIDColName string = "iql_txn_id" + maxTxnIDColName string = "iql_max_txn_id" + insIDColName string = "iql_insert_id" + insertEndodedColName string = "iql_insert_encoded" + latestUpdateColName string = "iql_last_modified" + gcStatusColName string = "iql_gc_status" +) + +type ControlAttributes interface { + GetControlGCStatusColumnName() string + GetControlGenIDColumnName() string + GetControlInsIDColumnName() string + GetControlInsertEncodedIDColumnName() string + GetControlLatestUpdateColumnName() string + GetControlMaxTxnColumnName() string + GetControlSsnIDColumnName() string + GetControlTxnIDColumnName() string +} + +func GetControlAttributes(attrType string) ControlAttributes { + return getControlAttributes(attrType) +} + +func getControlAttributes(attrType string) ControlAttributes { + switch strings.ToLower(attrType) { + case "standard": + return getStandardControlAttributes() + default: + return getStandardControlAttributes() + } +} + +func getStandardControlAttributes() ControlAttributes { + return &standardControlAttributes{} +} + +type standardControlAttributes struct{} + +func (ca *standardControlAttributes) GetControlGenIDColumnName() string { + return genIDColName +} + +func (ca *standardControlAttributes) GetControlSsnIDColumnName() string { + return ssnIDColName +} + +func (ca *standardControlAttributes) GetControlTxnIDColumnName() string { + return txnIDColName +} + +func (ca *standardControlAttributes) GetControlInsIDColumnName() string { + return insIDColName +} + +func (ca *standardControlAttributes) GetControlInsertEncodedIDColumnName() string { + return insertEndodedColName +} + +func (ca *standardControlAttributes) GetControlMaxTxnColumnName() string { + return maxTxnIDColName +} + +func (ca *standardControlAttributes) GetControlLatestUpdateColumnName() string { + return latestUpdateColName +} + +func (ca *standardControlAttributes) GetControlGCStatusColumnName() string { + return gcStatusColName +} diff --git a/internal/stackql/sqlengine/control/sqlcontrol_test.go b/internal/stackql/sqlengine/control/sqlcontrol_test.go new file mode 100644 index 000000000..65a37e443 --- /dev/null +++ b/internal/stackql/sqlengine/control/sqlcontrol_test.go @@ -0,0 +1,39 @@ +package sqlcontrol_test + +import ( + "testing" + + . "github.com/stackql/stackql/internal/stackql/sqlengine/control" + "github.com/stretchr/testify/assert" +) + +// TestGetControlAttributes tests the GetControlAttributes function. +func TestGetControlAttributes(t *testing.T) { + t.Run("standard type", func(t *testing.T) { + attrType := "standard" + controlAttrs := GetControlAttributes(attrType) + + assert.NotNil(t, controlAttrs, "Expected non-nil ControlAttributes for standard attrType") + }) + + t.Run("non-standard type", func(t *testing.T) { + attrType := "non-standard" + controlAttrs := GetControlAttributes(attrType) + + assert.NotNil(t, controlAttrs, "Expected non-nil ControlAttributes for non-standard attrType") + }) +} + +// TestStandardControlAttributes tests the standardControlAttributes' methods. +func TestStandardControlAttributes(t *testing.T) { + ca := GetControlAttributes("standard") + + assert.Equal(t, "iql_generation_id", ca.GetControlGenIDColumnName(), "Expected iql_generation_id") + assert.Equal(t, "iql_session_id", ca.GetControlSsnIDColumnName(), "Expected iql_session_id") + assert.Equal(t, "iql_txn_id", ca.GetControlTxnIDColumnName(), "Expected iql_txn_id") + assert.Equal(t, "iql_insert_id", ca.GetControlInsIDColumnName(), "Expected iql_insert_id") + assert.Equal(t, "iql_insert_encoded", ca.GetControlInsertEncodedIDColumnName(), "Expected iql_insert_encoded") + assert.Equal(t, "iql_max_txn_id", ca.GetControlMaxTxnColumnName(), "Expected iql_max_txn_id") + assert.Equal(t, "iql_last_modified", ca.GetControlLatestUpdateColumnName(), "Expected iql_last_modified") + assert.Equal(t, "iql_gc_status", ca.GetControlGCStatusColumnName(), "Expected iql_gc_status") +} diff --git a/internal/stackql/sqlengine/doc.go b/internal/stackql/sqlengine/doc.go new file mode 100644 index 000000000..84dbc52d0 --- /dev/null +++ b/internal/stackql/sqlengine/doc.go @@ -0,0 +1,8 @@ +// Package sqlengine owns StackQL's database connections and local SQL execution. +// Callers depend on SQLEngine; backend implementations and SQLite driver +// registration remain private to this package. +// +// Omni supplies provider rows without receiving a database connection. StackQL +// uses this package to stage those finished rows, evaluate the remaining SQL, +// and maintain local tables, views, and catalogue metadata. +package sqlengine diff --git a/internal/stackql/sqlengine/external.go b/internal/stackql/sqlengine/external.go new file mode 100644 index 000000000..62fc33c3d --- /dev/null +++ b/internal/stackql/sqlengine/external.go @@ -0,0 +1,107 @@ +package sqlengine + +import "gopkg.in/yaml.v3" + +var ( + _ SQLExternalConnection = standardSQLExternalConnection{} + _ SQLExternalTable = standardSQLExternalTable{} + _ SQLExternalColumn = standardSQLExternalColumn{} +) + +type SQLExternalConnection interface { + GetConnectionName() string + GetTables() map[string]SQLExternalTable +} + +type standardSQLExternalConnection struct { + ConnectionName string `json:"alias" yaml:"alias"` + Tables map[string]SQLExternalTable `json:"tables" yaml:"tables"` +} + +func (c standardSQLExternalConnection) GetConnectionName() string { + return c.ConnectionName +} + +func (c standardSQLExternalConnection) GetTables() map[string]SQLExternalTable { + return c.Tables +} + +type SQLExternalTable interface { + GetCatalogName() string + GetSchemaName() string + GetName() string + GetColumns() []SQLExternalColumn +} + +type standardSQLExternalTable struct { + CatalogName string `json:"catalogName" yaml:"catalogName"` + SchemaName string `json:"schemaName" yaml:"schemaName"` + Name string `json:"name" yaml:"name"` + Columns []*standardSQLExternalColumn `json:"columns" yaml:"columns"` +} + +func (t standardSQLExternalTable) GetCatalogName() string { + return t.CatalogName +} + +func (t standardSQLExternalTable) GetSchemaName() string { + return t.SchemaName +} + +func (t standardSQLExternalTable) GetName() string { + return t.Name +} + +func (t standardSQLExternalTable) GetColumns() []SQLExternalColumn { + var rv []SQLExternalColumn + for _, c := range t.Columns { + rv = append(rv, c) + } + return rv +} + +type SQLExternalColumn interface { + GetName() string + GetType() string + GetOid() uint32 + GetWidth() int + GetPrecision() int +} + +type standardSQLExternalColumn struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Oid uint32 `json:"oid" yaml:"oid"` + Width int `json:"width" yaml:"width"` + Precision int `json:"precision" yaml:"precision"` + // OrdinalPosition int `json:"ordinalPosition" yaml:"ordinalPosition"` +} + +func (c standardSQLExternalColumn) GetName() string { + return c.Name +} + +func (c standardSQLExternalColumn) GetType() string { + return c.Type +} + +func (c standardSQLExternalColumn) GetOid() uint32 { + return c.Oid +} + +func (c standardSQLExternalColumn) GetWidth() int { + return c.Width +} + +func (c standardSQLExternalColumn) GetPrecision() int { + return c.Precision +} + +// ExternalTable decodes a table declaration owned by the host SQL backend. +func ExternalTable(data []byte) (SQLExternalTable, error) { + var table standardSQLExternalTable + if err := yaml.Unmarshal(data, &table); err != nil { + return nil, err + } + return table, nil +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/DIVERGENCES.md b/internal/stackql/sqlengine/internal/sqlfuncs/DIVERGENCES.md new file mode 100644 index 000000000..ef6f37532 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/DIVERGENCES.md @@ -0,0 +1,148 @@ +# Behavioral divergences vs the retired C implementations + +This file documents every known behavioral difference between the pure Go +implementations in this package and the retired C implementations. The +authoritative C reference is the `SQLITE_ENABLE_STACKQL` amalgamation +additions in the stackql-go-sqlite3 fork - the code that actually shipped +in the retired cgo builds. github.com/stackql/sqlite-ext-functions is the +upstream source of most of that code, but the two are NOT identical: for +`aws_policy_equal` the fork carried two modifications that were never +backported to sqlite-ext-functions (see "aws_policy_equal comparison +rules" below). Anything not listed here is intended to be byte-for-byte +compatible with the fork's amalgamation additions and is pinned by the +golden vectors in `testdata/`. + +## Regexp functions (`regexp_like`, `regexp_substr`, `regexp_replace`) + +`regexp(pattern, source)` is a pure addition with no C counterpart: the +retired builds never registered it (the amalgamation's `regexp()` is +ICU-gated and was not compiled), so the SQL `REGEXP` operator documented in +the stackql language spec previously failed with "no such function". It is +`regexp_like` with swapped arguments, per the SQLite REGEXP-operator +contract, and shares every divergence below. + +The C implementations used tiny-regex-c (compiled with +`RE_DOT_MATCHES_NEWLINE=1`), a deliberately minimal engine. The Go +implementations use Go's `regexp` (RE2). The pattern language differences: + +1. **Groups `(...)` are now real groups.** tiny-regex-c treated `(` and `)` + as literal characters, so `abra(cad)?abra` could never match + `abracadabra`. RE2 interprets the group, so it matches. The affected + golden vectors (`regexp_like` `sef_10_01`, `regexp_replace` `sef_10_02`) + are recorded with the RE2 outcome and carry the original C outcome in + `retiredCResult`. The upstream test suite itself annotated these cases as + engine-conditional ("If (cad)? is unsupported, test returns 0"). +2. **Alternation `|` is now supported.** tiny-regex-c had its `BRANCH` + opcode disabled; `|` was a literal character. RE2 interprets it. +3. **Inverted character classes `[^...]` now work correctly.** They are + documented as broken in tiny-regex-c. +4. **No pattern-size ceiling.** tiny-regex-c silently failed to compile + patterns beyond ~30 nodes. RE2 has no comparable limit (default RE2 + program-size limits apply). +5. **Lazy quantifier semantics.** tiny-regex-c implemented `?` following a + quantifier with bespoke, partially non-standard semantics. RE2 implements + standard non-greedy quantifiers. +6. **Rune-oriented matching.** tiny-regex-c matched bytes; RE2 matches UTF-8 + runes. For ASCII patterns and subjects (the entire golden corpus) the + behavior is identical; multi-byte input may differ (e.g. `.` consumes one + rune, not one byte). +7. **Rejected patterns.** RE2 rejects backreferences and lookaround with an + error. tiny-regex-c did not support them either, but treated the + metacharacters literally instead of erroring. + +Preserved C behaviors (parity, pinned by vectors and tests): + +- `.` matches newline (`RE_DOT_MATCHES_NEWLINE=1` parity): patterns are + compiled with the `(?s)` flag. +- `regexp_replace` replacement text is **literal**: the C code copied the + replacement verbatim, so `$1`-style capture tokens are NOT expanded + (`ReplaceAllLiteralString`; vector `cref_literal_replacement`). +- NULL in any argument yields NULL; an unparseable pattern is an error. +- `regexp_substr` with no match yields NULL; an empty pattern matches the + empty string at position 0 and yields `''`. +- `regexp_replace` with no match returns the source unchanged. + +## JSON parsing (`json_equal`, `aws_policy_equal`) + +The C implementations parsed with cJSON; the Go implementations use +`encoding/json` with a strict single-document decode: + +8. **Trailing content is rejected.** cJSON's default parse entry point + ignored trailing bytes after the first JSON value (`{}garbage` parsed). + Go rejects trailing non-whitespace content with a parse error. +9. **Lax number forms are rejected.** cJSON tolerated some out-of-spec + number spellings and parsed overflowing numbers to infinity via + `strtod`. Go enforces RFC 8259 grammar and errors on numbers that + overflow a float64. +10. **Duplicate object keys.** cJSON kept every member and its lookups + matched the *first* occurrence; Go's `map[string]any` decoding keeps the + *last* occurrence. Documents with duplicate keys (invalid per RFC 8259 + "SHOULD be unique") may compare differently. + +Preserved C behaviors (parity): + +- Numbers compare as IEEE 754 doubles using cJSON's `DBL_EPSILON` rule + (`|a-b| <= max(|a|,|b|) * DBL_EPSILON`), so `1` equals `1.0` and one-ULP + differences are equal (vector `cref_number_epsilon`). +- `json_equal`: NULL arguments and parse failures are **errors**, not NULL + results. Object comparison is unordered with case-*sensitive* keys; array + comparison is ordered. +- `aws_policy_equal`: NULL arguments and parse failures are errors, except + that byte-identical inputs return 1 *before* parsing (the C `strcmp` + shortcut, vector `cref_strcmp_shortcut` - even non-JSON identical strings + compare equal). + +## aws_policy_equal comparison rules + +11. **Case-insensitive key lookup tie-breaking.** cJSON's + `cJSON_GetObjectItem` scanned members in insertion order and returned + the first case-insensitive match. Go maps do not preserve insertion + order, so the port resolves lookups deterministically: an exact-case + match wins, otherwise the lexicographically smallest case-folding-equal + key is used. Policies whose objects contain multiple keys differing only + in case (pathological input) may compare differently. + +Preserved C behaviors (parity, deliberately including its quirks): + +- The unordered-comparison field set is exactly `Action`, `NotAction`, + `Resource`, `NotResource`, `Principal`, `NotPrincipal`, `AWS`, `Service`, + `Tags`, `tags` (field name matched case-sensitively - `Tags` and `tags` + are two distinct entries). All other arrays - including `Statement` - + compare ordered. +- Two top-level array documents (an array at the root of both documents) + compare unordered. +- Within an unordered field, a string beginning with `arn:` compares ASCII + case-insensitively. The prefix test is applied to the **left** operand + only, preserving the C code's asymmetry. +- A single-element array equals its lone scalar element in either position. +- Unordered array elements are matched left-into-right after a length + check, without marking right-hand elements as consumed - arrays with + duplicate elements can double-match, exactly as in the C code. + +Fork-only modifications (authoritative contract): + +The stackql-go-sqlite3 fork's `SQLITE_ENABLE_STACKQL` amalgamation carried +two modifications to `aws_policy_equal` that were never backported to +github.com/stackql/sqlite-ext-functions. This package implements the fork +behavior, because the fork is what shipped in the retired cgo builds: + +- `Tags` and `tags` are members of the unordered-comparison field set + (vectors `fork_tags_top_*`, `fork_tags_statement_*`, + `fork_tags_lower_*`). +- Two top-level array documents (array vs array at the document root) are + compared unordered (vectors `fork_root_array_*`). A root array vs a + non-array root still compares under the ordinary rules (including the + single-element-array-equals-scalar rule). + +## Argument coercion + +12. **Text coercion of non-text arguments now happens in Go.** The C + implementations received arguments through `sqlite3_value_text()`. + The Go functions receive typed `driver.Value`s and apply the same + coercions themselves: integers format in base 10, REALs format in + shortest round-trip form with a `.0` suffix for integral values + (matching modern SQLite), blobs coerce byte-wise to text, and the + `split_part` index applies `sqlite3_value_int` semantics (NULL is 0, + REALs truncate toward zero, text parses as a leading integer prefix). + Exotic REAL edge cases could format differently than a given historical + SQLite build printed them. diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/awspolicyequal.go b/internal/stackql/sqlengine/internal/sqlfuncs/awspolicyequal.go new file mode 100644 index 000000000..6dcf004ad --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/awspolicyequal.go @@ -0,0 +1,169 @@ +package sqlfuncs + +import ( + "errors" + "sort" + "strings" +) + +// errInvalidPolicyArgs preserves the retired C error text for NULL arguments. +var errInvalidPolicyArgs = errors.New("Invalid policy strings") //nolint:staticcheck // C-parity error text + +// errPolicyParse preserves the retired C error text for unparseable documents. +var errPolicyParse = errors.New("Error parsing policy JSON strings") //nolint:staticcheck // C-parity error text + +// awsUnorderedFields are the policy fields whose arrays compare as unordered +// sets and whose "arn:" strings compare case-insensitively. "Tags" and "tags" +// are fork-only additions shipped in the stackql-go-sqlite3 fork's +// SQLITE_ENABLE_STACKQL amalgamation, never backported to +// sqlite-ext-functions (see DIVERGENCES.md). +// +//nolint:gochecknoglobals // Immutable policy field lookup. +var awsUnorderedFields = map[string]struct{}{ + "Action": {}, + "NotAction": {}, + "Resource": {}, + "NotResource": {}, + "Principal": {}, + "NotPrincipal": {}, + "AWS": {}, + "Service": {}, + "Tags": {}, + "tags": {}, +} + +// AWSPolicyEqual implements aws_policy_equal(a, b): 1 if the two IAM policy +// documents are equivalent under the retired C normalization rules (see +// awsPolicyCompare and DIVERGENCES.md), else 0. NULL or unparseable +// arguments are errors. +func AWSPolicyEqual(a, b any) (any, error) { + sa, okA := valueText(a) + sb, okB := valueText(b) + if !okA || !okB { + return nil, errInvalidPolicyArgs + } + if sa == sb { + return int64(1), nil + } + va, errA := parseJSONDocument(sa) + vb, errB := parseJSONDocument(sb) + if errA != nil || errB != nil { + return nil, errPolicyParse + } + // Fork-only behavior (see DIVERGENCES.md): two top-level array documents + // compare unordered, as in the stackql-go-sqlite3 fork's amalgamation. + _, aIsArray := va.([]any) + _, bIsArray := vb.([]any) + rootUnordered := aIsArray && bIsArray + if awsPolicyCompare(va, vb, rootUnordered) { + return int64(1), nil + } + return int64(0), nil +} + +// awsPolicyCompare ports aws_policy_compare_items: single-element arrays +// equal their lone string element, awsUnorderedFields arrays compare as sets, +// keys are case-insensitive, and the "arn:" case-fold is tested on the left +// operand only - a deliberate C asymmetry. +// +//nolint:gocognit,gocyclo,cyclop // Preserve the migrated policy comparison semantics. +func awsPolicyCompare(a, b any, parentUnordered bool) bool { + if aArr, ok := a.([]any); ok { + if bStr, isStr := b.(string); isStr { + return len(aArr) == 1 && awsPolicyCompare(aArr[0], bStr, parentUnordered) + } + } + if bArr, ok := b.([]any); ok { + if aStr, isStr := a.(string); isStr { + return len(bArr) == 1 && awsPolicyCompare(bArr[0], aStr, parentUnordered) + } + } + switch av := a.(type) { + case nil: + return b == nil + case bool: + bv, ok := b.(bool) + return ok && av == bv + case float64: + bv, ok := b.(float64) + return ok && compareDouble(av, bv) + case string: + bv, ok := b.(string) + if !ok { + return false + } + if parentUnordered && strings.HasPrefix(av, "arn:") { + return asciiEqualFold(av, bv) + } + return av == bv + case []any: + bv, ok := b.([]any) + if !ok || len(av) != len(bv) { + return false + } + if parentUnordered { + for _, aElem := range av { + if !awsFindMatchingElement(bv, aElem) { + return false + } + } + return true + } + for i := range av { + if !awsPolicyCompare(av[i], bv[i], parentUnordered) { + return false + } + } + return true + case map[string]any: + bv, ok := b.(map[string]any) + if !ok { + return false + } + for k, v := range av { + _, fieldUnordered := awsUnorderedFields[k] + bItem, present := awsObjectGet(bv, k) + if !present || !awsPolicyCompare(v, bItem, fieldUnordered) { + return false + } + } + for k := range bv { + if _, present := awsObjectGet(av, k); !present { + return false + } + } + return true + default: + return false + } +} + +// awsFindMatchingElement reports whether item matches any element of array +// under unordered comparison. +func awsFindMatchingElement(array []any, item any) bool { + for _, elem := range array { + if awsPolicyCompare(elem, item, true) { + return true + } + } + return false +} + +// awsObjectGet is a case-insensitive key lookup: an exact match wins, then +// the lexicographically smallest folding key, keeping lookups deterministic. +func awsObjectGet(obj map[string]any, key string) (any, bool) { + if v, ok := obj[key]; ok { + return v, true + } + var candidates []string + for k := range obj { + if asciiEqualFold(k, key) { + candidates = append(candidates, k) + } + } + if len(candidates) == 0 { + return nil, false + } + sort.Strings(candidates) + return obj[candidates[0]], true +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/coerce.go b/internal/stackql/sqlengine/internal/sqlfuncs/coerce.go new file mode 100644 index 000000000..b65173213 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/coerce.go @@ -0,0 +1,118 @@ +package sqlfuncs + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// dblEpsilon is C's DBL_EPSILON, the tolerance the retired cJSON-based +// implementations used for number equality. +const dblEpsilon = 2.220446049250313e-16 + +// valueText mirrors sqlite3_value_text() coercion; the boolean reports +// whether the value was non-NULL. +func valueText(v any) (string, bool) { + switch t := v.(type) { + case nil: + return "", false + case string: + return t, true + case []byte: + return string(t), true + case int64: + return strconv.FormatInt(t, 10), true + case float64: + return formatFloatText(t), true + default: + return fmt.Sprintf("%v", t), true + } +} + +// formatFloatText renders a float the way SQLite renders REAL as text: +// shortest round-trip form always carrying a decimal point or exponent. +func formatFloatText(f float64) string { + s := strconv.FormatFloat(f, 'g', -1, 64) + if !strings.ContainsAny(s, ".eE") && !math.IsInf(f, 0) && !math.IsNaN(f) { + s += ".0" + } + return s +} + +// valueInt mirrors sqlite3_value_int() coercion: NULL is 0, floats truncate +// toward zero, text parses as a signed leading-digit prefix. +func valueInt(v any) int64 { + switch t := v.(type) { + case nil: + return 0 + case int64: + return t + case float64: + if math.IsNaN(t) { + return 0 + } + return int64(math.Trunc(t)) + case string: + return parseIntPrefix(t) + case []byte: + return parseIntPrefix(string(t)) + default: + return 0 + } +} + +// parseIntPrefix parses an optionally signed run of leading ASCII digits +// after whitespace, the way SQLite coerces text to an integer. +func parseIntPrefix(s string) int64 { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r' || s[i] == '\v' || s[i] == '\f') { + i++ + } + start := i + if i < len(s) && (s[i] == '+' || s[i] == '-') { + i++ + } + digitsStart := i + for i < len(s) && s[i] >= '0' && s[i] <= '9' { + i++ + } + if i == digitsStart { + return 0 + } + n, err := strconv.ParseInt(s[start:i], 10, 64) + if err != nil { + return 0 + } + return n +} + +// compareDouble is the retired implementations' epsilon comparison: +// |a-b| <= max(|a|,|b|) * DBL_EPSILON. +func compareDouble(a, b float64) bool { + maxVal := math.Abs(a) + if abs := math.Abs(b); abs > maxVal { + maxVal = abs + } + return math.Abs(a-b) <= maxVal*dblEpsilon +} + +// asciiEqualFold reports byte-wise ASCII case-insensitive equality. +func asciiEqualFold(a, b string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + ca, cb := a[i], b[i] + if 'A' <= ca && ca <= 'Z' { + ca += 'a' - 'A' + } + if 'A' <= cb && cb <= 'Z' { + cb += 'a' - 'A' + } + if ca != cb { + return false + } + } + return true +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/fuzz_test.go b/internal/stackql/sqlengine/internal/sqlfuncs/fuzz_test.go new file mode 100644 index 000000000..29b2bb90c --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/fuzz_test.go @@ -0,0 +1,178 @@ +package sqlfuncs //nolint:testpackage // Exercises private scalar conversion helpers. + +import ( + "regexp" + "strings" + "testing" +) + +func FuzzSplitPart(f *testing.F) { + f.Add("a/b/c", "/", int64(2)) + f.Add("/a/b/c/", "/", int64(-1)) + f.Add("", "/", int64(1)) + f.Add("a,b,c", ",", int64(0)) + f.Add("aaa", "aa", int64(1)) + f.Fuzz(func(t *testing.T, source, sep string, part int64) { + got, err := SplitPart(source, sep, part) + if err != nil { + t.Fatalf("split_part must never error, got %v", err) + } + switch v := got.(type) { + case nil: + if sep != "" && part >= 1 && part <= int64(strings.Count(source, sep))+1 { + t.Fatalf("split_part(%q, %q, %d) returned NULL for an in-range part", source, sep, part) + } + case string: + if sep == "" { + t.Fatalf("split_part with empty separator must be NULL, got %q", v) + } + if !strings.Contains(source, v) { + t.Fatalf("split_part(%q, %q, %d) = %q is not a substring of the source", source, sep, part, v) + } + default: + t.Fatalf("split_part returned unexpected type %T", got) + } + }) +} + +// fuzzCompiles reports whether the pattern compiles under the production +// translation; fuzz targets use it to decide whether an error is legitimate. +func fuzzCompiles(pattern string) bool { + _, err := regexp.Compile("(?s)" + pattern) + return err == nil +} + +func FuzzRegexpLike(f *testing.F) { + f.Add("hello world", "hello") + f.Add("abracadabra", "abra(cad)?abra") + f.Add("abc123", "[0-9]+") + f.Add("a\nb", "a.b") + f.Add("", "") + f.Fuzz(func(t *testing.T, source, pattern string) { + got, err := RegexpLike(source, pattern) + if err != nil { + if fuzzCompiles(pattern) { + t.Fatalf("regexp_like(%q, %q) errored on a valid pattern: %v", source, pattern, err) + } + return + } + if got != int64(0) && got != int64(1) { + t.Fatalf("regexp_like(%q, %q) = %#v, want 0 or 1", source, pattern, got) + } + }) +} + +func FuzzRegexp(f *testing.F) { + f.Add("hello", "hello world") + f.Add("[0-9]+", "the year is 2021") + f.Fuzz(func(t *testing.T, pattern, source string) { + got, err := Regexp(pattern, source) + if err != nil { + if fuzzCompiles(pattern) { + t.Fatalf("regexp(%q, %q) errored on a valid pattern: %v", pattern, source, err) + } + return + } + want, wantErr := RegexpLike(source, pattern) + if wantErr != nil || got != want { + t.Fatalf("regexp(%q, %q) = %#v, want regexp_like result %#v (err %v)", pattern, source, got, want, wantErr) + } + }) +} + +func FuzzRegexpSubstr(f *testing.F) { + f.Add("hello world", "w.*d") + f.Add("file.txt", "\\.\\w+$") + f.Add("abc", "[0-9]") + f.Add("hello", "") + f.Fuzz(func(t *testing.T, source, pattern string) { + got, err := RegexpSubstr(source, pattern) + if err != nil { + if fuzzCompiles(pattern) { + t.Fatalf("regexp_substr(%q, %q) errored on a valid pattern: %v", source, pattern, err) + } + return + } + switch v := got.(type) { + case nil: + case string: + if !strings.Contains(source, v) { + t.Fatalf("regexp_substr(%q, %q) = %q is not a substring of the source", source, pattern, v) + } + default: + t.Fatalf("regexp_substr returned unexpected type %T", got) + } + }) +} + +func FuzzRegexpReplace(f *testing.F) { + f.Add("hello world", "world", "SQLite") + f.Add("123-456-7890", "[0-9]", "X") + f.Add("abracadabra", "abra(cad)?abra", "magic") + f.Add("ab12cd", "([a-z]+)", "<$1>") + f.Fuzz(func(t *testing.T, source, pattern, replacement string) { + got, err := RegexpReplace(source, pattern, replacement) + if err != nil { + if fuzzCompiles(pattern) { + t.Fatalf("regexp_replace(%q, %q, %q) errored on a valid pattern: %v", source, pattern, replacement, err) + } + return + } + if _, ok := got.(string); !ok { + t.Fatalf("regexp_replace returned unexpected type %T", got) + } + }) +} + +func FuzzJSONEqual(f *testing.F) { + f.Add(`{"key": "value"}`, `{ "key" : "value" }`) + f.Add(`[1, 2, 3]`, `[3, 2, 1]`) + f.Add(`{"a":1}`, `{ "a" : 1.0 }`) + f.Add(`1`, `2`) + f.Add(`not json`, `{}`) + f.Fuzz(func(t *testing.T, a, b string) { + got, err := JSONEqual(a, b) + mirror, mirrorErr := JSONEqual(b, a) + if (err != nil) != (mirrorErr != nil) { + t.Fatalf("json_equal error asymmetry: (%q,%q) err=%v, mirrored err=%v", a, b, err, mirrorErr) + } + if err != nil { + return + } + if got != int64(0) && got != int64(1) { + t.Fatalf("json_equal(%q, %q) = %#v, want 0 or 1", a, b, got) + } + if got != mirror { + t.Fatalf("json_equal symmetry broken: (%q,%q)=%v but mirrored=%v", a, b, got, mirror) + } + if self, selfErr := JSONEqual(a, a); selfErr == nil && self != int64(1) { + t.Fatalf("json_equal(%q, %q) must be reflexive", a, a) + } + }) +} + +func FuzzAWSPolicyEqual(f *testing.F) { + f.Add(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":"*"}]}`, + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`) + f.Add(`{"Resource":"arn:aws:s3:::mybucket"}`, `{"Resource":"arn:aws:S3:::mybucket"}`) + f.Add(`not json`, `not json`) + f.Add(`{"a": [1]}`, `{"A": 1}`) + f.Fuzz(func(t *testing.T, a, b string) { + // Note: no symmetry assertion - the "arn:" case-insensitive rule is + // deliberately tested on the left operand only, matching the C code. + got, err := AWSPolicyEqual(a, b) + if err != nil { + return + } + if got != int64(0) && got != int64(1) { + t.Fatalf("aws_policy_equal(%q, %q) = %#v, want 0 or 1", a, b, got) + } + self, selfErr := AWSPolicyEqual(a, a) + if selfErr != nil { + t.Fatalf("aws_policy_equal(%q, %q) must not error (identical-string shortcut): %v", a, a, selfErr) + } + if self != int64(1) { + t.Fatalf("aws_policy_equal(%q, %q) must be reflexive", a, a) + } + }) +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/jsonequal.go b/internal/stackql/sqlengine/internal/sqlfuncs/jsonequal.go new file mode 100644 index 000000000..e22c29f9e --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/jsonequal.go @@ -0,0 +1,91 @@ +package sqlfuncs + +import ( + "encoding/json" + "errors" + "strings" +) + +// errInvalidJSONArgs preserves the retired C error text for NULL arguments. +var errInvalidJSONArgs = errors.New("Invalid JSON strings") //nolint:staticcheck // C-parity error text + +// errJSONParse preserves the retired C error text for unparseable documents. +var errJSONParse = errors.New("Error parsing JSON strings") //nolint:staticcheck // C-parity error text + +// JSONEqual implements json_equal(a, b): 1 if the documents are deeply equal, +// else 0. Objects compare unordered, arrays ordered, numbers as doubles with +// the epsilon rule (1 == 1.0). NULL or unparseable arguments are errors. +func JSONEqual(a, b any) (any, error) { + sa, okA := valueText(a) + sb, okB := valueText(b) + if !okA || !okB { + return nil, errInvalidJSONArgs + } + va, errA := parseJSONDocument(sa) + vb, errB := parseJSONDocument(sb) + if errA != nil || errB != nil { + return nil, errJSONParse + } + if jsonDeepEqual(va, vb) { + return int64(1), nil + } + return int64(0), nil +} + +// parseJSONDocument parses exactly one JSON value, rejecting trailing +// non-whitespace content (a divergence from cJSON, see DIVERGENCES.md). +func parseJSONDocument(s string) (any, error) { + dec := json.NewDecoder(strings.NewReader(s)) + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if dec.More() { + return nil, errors.New("trailing content after JSON value") + } + return v, nil +} + +// jsonDeepEqual ports cJSON_Compare with case-sensitive keys. +// +//nolint:gocognit // Preserve the migrated JSON comparison semantics. +func jsonDeepEqual(a, b any) bool { + switch av := a.(type) { + case nil: + return b == nil + case bool: + bv, ok := b.(bool) + return ok && av == bv + case float64: + bv, ok := b.(float64) + return ok && compareDouble(av, bv) + case string: + bv, ok := b.(string) + return ok && av == bv + case []any: + bv, ok := b.([]any) + if !ok || len(av) != len(bv) { + return false + } + for i := range av { + if !jsonDeepEqual(av[i], bv[i]) { + return false + } + } + return true + case map[string]any: + bv, ok := b.(map[string]any) + if !ok || len(av) != len(bv) { + return false + } + for k, v := range av { + bItem, present := bv[k] + if !present || !jsonDeepEqual(v, bItem) { + return false + } + } + return true + default: + return false + } +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/regexp.go b/internal/stackql/sqlengine/internal/sqlfuncs/regexp.go new file mode 100644 index 000000000..0ae30b9e9 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/regexp.go @@ -0,0 +1,76 @@ +package sqlfuncs + +import ( + "fmt" + "regexp" +) + +// compilePattern compiles pattern in (?s) mode: the retired C engine was +// built with RE_DOT_MATCHES_NEWLINE=1, so '.' must match any byte. +func compilePattern(pattern string) (*regexp.Regexp, error) { + re, err := regexp.Compile("(?s)" + pattern) + if err != nil { + return nil, fmt.Errorf("invalid regular expression: %w", err) + } + return re, nil +} + +// RegexpLike implements regexp_like(source, pattern): 1 on match, else 0. +// NULL source or pattern yields NULL; an invalid pattern is an error. +func RegexpLike(source, pattern any) (any, error) { + src, srcOK := valueText(source) + pat, patOK := valueText(pattern) + if !srcOK || !patOK { + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. + } + re, err := compilePattern(pat) + if err != nil { + return nil, err + } + if re.MatchString(src) { + return int64(1), nil + } + return int64(0), nil +} + +// Regexp implements regexp(pattern, source), the function SQLite invokes for +// the REGEXP operator: `X REGEXP Y` evaluates regexp(Y, X). +func Regexp(pattern, source any) (any, error) { + return RegexpLike(source, pattern) +} + +// RegexpSubstr implements regexp_substr(source, pattern): the leftmost match +// of pattern in source, or NULL if none. NULL source or pattern yields NULL. +func RegexpSubstr(source, pattern any) (any, error) { + src, srcOK := valueText(source) + pat, patOK := valueText(pattern) + if !srcOK || !patOK { + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. + } + re, err := compilePattern(pat) + if err != nil { + return nil, err + } + loc := re.FindStringIndex(src) + if loc == nil { + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. + } + return src[loc[0]:loc[1]], nil +} + +// RegexpReplace implements regexp_replace(source, pattern, replacement): +// replaces every match with replacement taken literally ($1 is not expanded, +// matching the retired C implementation). Any NULL argument yields NULL. +func RegexpReplace(source, pattern, replacement any) (any, error) { + src, srcOK := valueText(source) + pat, patOK := valueText(pattern) + rep, repOK := valueText(replacement) + if !srcOK || !patOK || !repOK { + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. + } + re, err := compilePattern(pat) + if err != nil { + return nil, err + } + return re.ReplaceAllLiteralString(src, rep), nil +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/register.go b/internal/stackql/sqlengine/internal/sqlfuncs/register.go new file mode 100644 index 000000000..9a4ee346b --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/register.go @@ -0,0 +1,58 @@ +package sqlfuncs + +import ( + "database/sql/driver" + "errors" + + sqlite "modernc.org/sqlite" +) + +// Register registers the StackQL extension functions as deterministic scalar +// functions on the supplied driver. It is the only driver-facing entry point +// of this package. +func Register(drv *sqlite.Driver) error { + if drv == nil { + return errors.New("sqlfuncs: cannot register functions on a nil driver") + } + registrations := []struct { + name string + nArgs int32 + impl func(args []driver.Value) (driver.Value, error) + }{ + {"split_part", 3, func(args []driver.Value) (driver.Value, error) { + return SplitPart(args[0], args[1], args[2]) + }}, + {"regexp_like", 2, func(args []driver.Value) (driver.Value, error) { + return RegexpLike(args[0], args[1]) + }}, + {"regexp", 2, func(args []driver.Value) (driver.Value, error) { + return Regexp(args[0], args[1]) + }}, + {"regexp_substr", 2, func(args []driver.Value) (driver.Value, error) { + return RegexpSubstr(args[0], args[1]) + }}, + {"regexp_replace", 3, func(args []driver.Value) (driver.Value, error) { + return RegexpReplace(args[0], args[1], args[2]) + }}, + {"json_equal", 2, func(args []driver.Value) (driver.Value, error) { + return JSONEqual(args[0], args[1]) + }}, + {"aws_policy_equal", 2, func(args []driver.Value) (driver.Value, error) { + return AWSPolicyEqual(args[0], args[1]) + }}, + } + for _, reg := range registrations { + impl := reg.impl + err := drv.RegisterDeterministicScalarFunction( + reg.name, + reg.nArgs, + func(_ *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) { + return impl(args) + }, + ) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/splitpart.go b/internal/stackql/sqlengine/internal/sqlfuncs/splitpart.go new file mode 100644 index 000000000..6c1994f75 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/splitpart.go @@ -0,0 +1,26 @@ +package sqlfuncs + +import "strings" + +// SplitPart implements split_part(source, separator, part): the one-based +// part of source split on separator; negative indexes count from the end. +// NULL source, NULL or empty separator, and out-of-range indexes yield NULL. +func SplitPart(source, separator, part any) (any, error) { + src, srcOK := valueText(source) + sep, sepOK := valueText(separator) + if !srcOK || !sepOK || sep == "" { + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. + } + idx := valueInt(part) + parts := strings.Split(src, sep) + n := int64(len(parts)) + if idx < 0 { + idx = n + idx + } else { + idx-- + } + if idx >= 0 && idx < n { + return parts[idx], nil + } + return nil, nil //nolint:nilnil // A SQL NULL is a valid scalar result. +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/aws_policy_equal.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/aws_policy_equal.json new file mode 100644 index 000000000..7d9b2553c --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/aws_policy_equal.json @@ -0,0 +1,254 @@ +{ + "function": "aws_policy_equal", + "vectors": [ + { + "id": "sef_1_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 1_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}]}" + ], + "result": 1 + }, + { + "id": "sef_2_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 2_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"},{\"Effect\":\"Deny\",\"Action\":\"ec2:*\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Action\":\"ec2:*\",\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}]}" + ], + "result": 0, + "note": "Statement arrays are order-sensitive" + }, + { + "id": "sef_2_02", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 2_02", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Action\":\"s3:*\",\"Resource\":\"*\"}]}" + ], + "result": 0 + }, + { + "id": "sef_3_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 3_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\"],\"Resource\":\"*\"}]}" + ], + "result": 1 + }, + { + "id": "sef_3_02", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 3_02", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":[\"arn:aws:s3:::mybucket\",\"arn:aws:s3:::otherbucket\"]}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":[\"arn:aws:s3:::otherbucket\",\"arn:aws:s3:::mybucket\"]}]}" + ], + "result": 1 + }, + { + "id": "sef_4_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 4_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::123456789012:role/role1\",\"arn:aws:iam::123456789012:role/role2\"]},\"Action\":\"s3:*\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::123456789012:role/role2\",\"arn:aws:iam::123456789012:role/role1\"]},\"Action\":\"s3:*\",\"Resource\":\"*\"}]}" + ], + "result": 1 + }, + { + "id": "sef_4_02", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 4_02", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::123456789012:role/role1\",\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"s3:*\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\",\"AWS\":\"arn:aws:iam::123456789012:role/role1\"},\"Action\":\"s3:*\",\"Resource\":\"*\"}]}" + ], + "result": 1 + }, + { + "id": "sef_5_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 5_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::123456789012:role/role1\"]},\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Resource\":[\"arn:aws:s3:::bucket1/*\",\"arn:aws:s3:::bucket2/*\"],\"Condition\":{\"StringEquals\":{\"aws:PrincipalTag/department\":\"HR\"}}}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::123456789012:role/role1\"]},\"Action\":[\"s3:PutObject\",\"s3:GetObject\"],\"Resource\":[\"arn:aws:s3:::bucket2/*\",\"arn:aws:s3:::bucket1/*\"],\"Condition\":{\"StringEquals\":{\"aws:PrincipalTag/department\":\"HR\"}}}]}" + ], + "result": 1 + }, + { + "id": "sef_6_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 6_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Condition\":{\"StringEquals\":{\"sts:ExternalId\":\"0000\"}},\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL\"}}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL\"},\"Action\":\"sts:AssumeRole\",\"Condition\":{\"StringEquals\":{\"sts:ExternalId\":\"0000\"}}}]}" + ], + "result": 1 + }, + { + "id": "sef_7_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 7_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Statement1\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeInstances\",\"ec2:StartInstances\"],\"Resource\":\"*\"},{\"Sid\":\"Statement2\",\"Effect\":\"Deny\",\"Action\":\"ec2:StopInstances\",\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"Statement1\",\"Effect\":\"Allow\",\"Action\":[\"ec2:StartInstances\",\"ec2:DescribeInstances\"],\"Resource\":\"*\"},{\"Sid\":\"Statement2\",\"Effect\":\"Deny\",\"Action\":\"ec2:StopInstances\",\"Resource\":\"*\"}]}" + ], + "result": 1 + }, + { + "id": "sef_8_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 8_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"*\",\"Resource\":\"arn:aws:s3:::mybucket\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"*\",\"Resource\":\"arn:aws:S3:::mybucket\"}]}" + ], + "result": 1, + "note": "ARN strings under unordered fields compare case-insensitively" + }, + { + "id": "sef_9_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 9_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:GetObject\"],\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":\"*\"}]}" + ], + "result": 1, + "note": "single-element array form equals scalar form" + }, + { + "id": "sef_10_01", + "source": "sqlite-ext-functions test/aws_policy_equal.sql 10_01", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Condition\":{\"StringEquals\":{\"aws:username\":\"johndoe\"},\"Bool\":{\"aws:SecureTransport\":\"true\"}}}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"true\"},\"StringEquals\":{\"aws:username\":\"johndoe\"}}}]}" + ], + "result": 1 + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:GetObject\"],\"Resource\":\"*\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":\"*\"}]}" + ], + "result": 1, + "note": "scalar vs array forms sample" + }, + { + "id": "cref_null_lhs", + "source": "C reference implementation behavior (aws_policy_equal.c)", + "args": [ + null, + "{}" + ], + "error": true, + "note": "NULL arguments raise an error in the C implementation, not NULL" + }, + { + "id": "cref_null_rhs", + "source": "C reference implementation behavior (aws_policy_equal.c)", + "args": [ + "{}", + null + ], + "error": true + }, + { + "id": "cref_strcmp_shortcut", + "source": "C reference implementation behavior (aws_policy_equal.c)", + "args": [ + "not json", + "not json" + ], + "result": 1, + "note": "byte-identical strings compare equal before any parsing" + }, + { + "id": "cref_invalid_json", + "source": "C reference implementation behavior (aws_policy_equal.c)", + "args": [ + "not json", + "not json" + ], + "error": true, + "note": "non-identical unparseable documents raise a parse error" + }, + { + "id": "fork_tags_top_01", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}],\"Tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}],\"Tags\":[{\"Key\":\"team\",\"Value\":\"data\"},{\"Key\":\"env\",\"Value\":\"prod\"}]}" + ], + "result": 1, + "note": "top-level Tags arrays compare unordered (fork-only field-set addition)" + }, + { + "id": "fork_tags_top_02", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}],\"Tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\"}],\"Tags\":[{\"Key\":\"env\",\"Value\":\"dev\"},{\"Key\":\"team\",\"Value\":\"data\"}]}" + ], + "result": 0, + "note": "top-level Tags with a differing element are unequal" + }, + { + "id": "fork_tags_statement_01", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Tags\":[{\"Key\":\"team\",\"Value\":\"data\"},{\"Key\":\"env\",\"Value\":\"prod\"}]}]}" + ], + "result": 1, + "note": "Tags arrays nested within a statement compare unordered (fork-only field-set addition)" + }, + { + "id": "fork_tags_statement_02", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}]}", + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"*\",\"Tags\":[{\"Key\":\"env\",\"Value\":\"dev\"},{\"Key\":\"team\",\"Value\":\"data\"}]}]}" + ], + "result": 0, + "note": "nested Tags with a differing element are unequal" + }, + { + "id": "fork_tags_lower_01", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}", + "{\"Version\":\"2012-10-17\",\"tags\":[{\"Key\":\"team\",\"Value\":\"data\"},{\"Key\":\"env\",\"Value\":\"prod\"}]}" + ], + "result": 1, + "note": "lowercase tags is a distinct fork-only field-set entry (set matching is case-sensitive)" + }, + { + "id": "fork_tags_lower_02", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "{\"Version\":\"2012-10-17\",\"tags\":[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]}", + "{\"Version\":\"2012-10-17\",\"tags\":[{\"Key\":\"env\",\"Value\":\"dev\"},{\"Key\":\"team\",\"Value\":\"data\"}]}" + ], + "result": 0, + "note": "lowercase tags with a differing element are unequal" + }, + { + "id": "fork_root_array_01", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]", + "[{\"Key\":\"team\",\"Value\":\"data\"},{\"Key\":\"env\",\"Value\":\"prod\"}]" + ], + "result": 1, + "note": "two top-level array documents compare unordered (fork-only behavior)" + }, + { + "id": "fork_root_array_02", + "source": "stackql-go-sqlite3 fork amalgamation (SQLITE_ENABLE_STACKQL) fork-only behavior", + "args": [ + "[{\"Key\":\"env\",\"Value\":\"prod\"},{\"Key\":\"team\",\"Value\":\"data\"}]", + "[{\"Key\":\"env\",\"Value\":\"dev\"},{\"Key\":\"team\",\"Value\":\"data\"}]" + ], + "result": 0, + "note": "top-level array documents with a differing element are unequal" + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/json_equal.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/json_equal.json new file mode 100644 index 000000000..0bae5a05a --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/json_equal.json @@ -0,0 +1,225 @@ +{ + "function": "json_equal", + "vectors": [ + { + "id": "sef_2_01", + "source": "sqlite-ext-functions test/json_equal.sql 2_01", + "args": [ + "{\"key\": \"value\"}", + "{\"key\": \"value\"}" + ], + "result": 1 + }, + { + "id": "sef_2_02", + "source": "sqlite-ext-functions test/json_equal.sql 2_02", + "args": [ + "{\"key1\": \"value1\", \"key2\": \"value2\"}", + "{\"key1\": \"value1\", \"key2\": \"value2\"}" + ], + "result": 1 + }, + { + "id": "sef_3_01", + "source": "sqlite-ext-functions test/json_equal.sql 3_01", + "args": [ + "{\"key1\": \"value1\", \"key2\": \"value2\"}", + "{\"key2\": \"value2\", \"key1\": \"value1\"}" + ], + "result": 1 + }, + { + "id": "sef_4_01", + "source": "sqlite-ext-functions test/json_equal.sql 4_01", + "args": [ + "{\"key\": \"value\"}", + "{\"key\": \"different\"}" + ], + "result": 0 + }, + { + "id": "sef_4_02", + "source": "sqlite-ext-functions test/json_equal.sql 4_02", + "args": [ + "{\"key1\": \"value1\"}", + "{\"key2\": \"value2\"}" + ], + "result": 0 + }, + { + "id": "sef_5_01", + "source": "sqlite-ext-functions test/json_equal.sql 5_01", + "args": [ + "[1, 2, 3]", + "[1, 2, 3]" + ], + "result": 1 + }, + { + "id": "sef_5_02", + "source": "sqlite-ext-functions test/json_equal.sql 5_02", + "args": [ + "[{\"key\": \"value\"}]", + "[{\"key\": \"value\"}]" + ], + "result": 1 + }, + { + "id": "sef_6_01", + "source": "sqlite-ext-functions test/json_equal.sql 6_01", + "args": [ + "[1, 2, 3]", + "[3, 2, 1]" + ], + "result": 0 + }, + { + "id": "sef_6_02", + "source": "sqlite-ext-functions test/json_equal.sql 6_02", + "args": [ + "[{\"key\": \"value\"}]", + "[{\"key\": \"different\"}]" + ], + "result": 0 + }, + { + "id": "sef_7_01", + "source": "sqlite-ext-functions test/json_equal.sql 7_01", + "args": [ + "{\"key\": [1, 2, {\"subkey\": \"subvalue\"}]}", + "{\"key\": [1, 2, {\"subkey\": \"subvalue\"}]}" + ], + "result": 1 + }, + { + "id": "sef_7_02", + "source": "sqlite-ext-functions test/json_equal.sql 7_02", + "args": [ + "{\"key\": [1, 2, {\"subkey\": \"subvalue1\"}]}", + "{\"key\": [1, 2, {\"subkey\": \"subvalue2\"}]}" + ], + "result": 0 + }, + { + "id": "sef_8_01", + "source": "sqlite-ext-functions test/json_equal.sql 8_01", + "args": [ + "{\"key\": \"value\"}", + " { \"key\" : \"value\" } " + ], + "result": 1 + }, + { + "id": "sef_8_02", + "source": "sqlite-ext-functions test/json_equal.sql 8_02", + "args": [ + "[1, 2, 3]", + "[ 1, 2, 3 ]" + ], + "result": 1 + }, + { + "id": "sef_9_01", + "source": "sqlite-ext-functions test/json_equal.sql 9_01", + "args": [ + "{\"key\": \"value with space\"}", + "{\"key\": \"value with space\"}" + ], + "result": 1 + }, + { + "id": "sef_9_02", + "source": "sqlite-ext-functions test/json_equal.sql 9_02", + "args": [ + "{\"key\": \"value\\nwith\\nnewlines\"}", + "{\"key\": \"value\\nwith\\nnewlines\"}" + ], + "result": 1, + "note": "the SQL literal carries a two-character backslash-n JSON escape; both documents parse to real newlines" + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "{\"a\":1}", + "{ \"a\" : 1.0 }" + ], + "result": 1 + }, + { + "id": "cref_null_lhs", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + null, + "{}" + ], + "error": true, + "note": "NULL arguments raise an error in the C implementation, not NULL" + }, + { + "id": "cref_null_rhs", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "{}", + null + ], + "error": true + }, + { + "id": "cref_invalid_json", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "not json", + "{}" + ], + "error": true, + "note": "unparseable documents raise an error in the C implementation" + }, + { + "id": "cref_scalar_equal", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "1", + "1" + ], + "result": 1 + }, + { + "id": "cref_scalar_unequal", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "1", + "2" + ], + "result": 0 + }, + { + "id": "cref_number_epsilon", + "source": "C reference implementation behavior (json_equal.c compare_double)", + "args": [ + "{\"a\": 1}", + "{\"a\": 1.0000000000000002}" + ], + "result": 1, + "note": "one ULP at 1.0 is within the cJSON DBL_EPSILON comparison" + }, + { + "id": "cref_bool_null", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "{\"a\": true, \"b\": null}", + "{\"b\": null, \"a\": true}" + ], + "result": 1 + }, + { + "id": "cref_type_mismatch", + "source": "C reference implementation behavior (json_equal.c)", + "args": [ + "{\"a\": 1}", + "{\"a\": \"1\"}" + ], + "result": 0 + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp.json new file mode 100644 index 000000000..a33b5cebf --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp.json @@ -0,0 +1,142 @@ +{ + "function": "regexp", + "group_note": "regexp(pattern, source) backs the SQL REGEXP operator and is regexp_like(source, pattern) with swapped arguments; vectors are derived from the regexp_like golden suite plus the stackql.io REGEXP doc examples.", + "vectors": [ + { + "id": "derived_1_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 1_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "pattern", + null + ], + "result": null + }, + { + "id": "derived_1_02", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 1_02 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + null, + "source" + ], + "result": null + }, + { + "id": "derived_2_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 2_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "hello", + "hello world" + ], + "result": 1 + }, + { + "id": "derived_2_02", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 2_02 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "^world", + "hello world" + ], + "result": 0 + }, + { + "id": "derived_2_03", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 2_03 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "world$", + "hello world" + ], + "result": 1 + }, + { + "id": "derived_3_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 3_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "hello\\.world", + "hello.world" + ], + "result": 1 + }, + { + "id": "derived_3_02", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 3_02 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "[0-9]+", + "abc123" + ], + "result": 1 + }, + { + "id": "derived_3_03", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 3_03 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "[0-9]+", + "abc" + ], + "result": 0 + }, + { + "id": "derived_10_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 10_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "abra(cad)?abra", + "abracadabra" + ], + "result": 1, + "note": "tiny-regex-c (the retired C engine) did not support groups: '(' and ')' were literal characters, so this pattern could not match and the C suite recorded the opposite outcome (see retiredCResult). Go RE2 supports groups; expectation deliberately updated per DIVERGENCES.md." + }, + { + "id": "derived_11_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 11_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "", + "hello" + ], + "result": 1 + }, + { + "id": "derived_12_01", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 12_01 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "h.llo", + "hello" + ], + "result": 1 + }, + { + "id": "derived_12_02", + "source": "arg-swapped from sqlite-ext-functions test/regexp.sql 12_02 (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "\\d+\\w+", + "123abc" + ], + "result": 1 + }, + { + "id": "pr783_01", + "source": "arg-swapped from stackql PR #783 cgo golden sample (SQLite REGEXP operator contract: X REGEXP Y == regexp(Y, X))", + "args": [ + "\\d+", + "abc123" + ], + "result": 1 + }, + { + "id": "stackqlio_regexp_md_01", + "source": "stackql.io docs/language-spec/functions/re/regexp.md example 1", + "args": [ + "[0-9]+", + "the year is 2021" + ], + "result": 1 + }, + { + "id": "stackqlio_regexp_md_02", + "source": "stackql.io docs/language-spec/functions/re/regexp.md example 2", + "args": [ + "^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$", + "user@example.com" + ], + "result": 1 + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_like.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_like.json new file mode 100644 index 000000000..fddfc8142 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_like.json @@ -0,0 +1,124 @@ +{ + "function": "regexp_like", + "vectors": [ + { + "id": "sef_1_01", + "source": "sqlite-ext-functions test/regexp.sql 1_01", + "args": [ + null, + "pattern" + ], + "result": null + }, + { + "id": "sef_1_02", + "source": "sqlite-ext-functions test/regexp.sql 1_02", + "args": [ + "source", + null + ], + "result": null + }, + { + "id": "sef_2_01", + "source": "sqlite-ext-functions test/regexp.sql 2_01", + "args": [ + "hello world", + "hello" + ], + "result": 1 + }, + { + "id": "sef_2_02", + "source": "sqlite-ext-functions test/regexp.sql 2_02", + "args": [ + "hello world", + "^world" + ], + "result": 0 + }, + { + "id": "sef_2_03", + "source": "sqlite-ext-functions test/regexp.sql 2_03", + "args": [ + "hello world", + "world$" + ], + "result": 1 + }, + { + "id": "sef_3_01", + "source": "sqlite-ext-functions test/regexp.sql 3_01", + "args": [ + "hello.world", + "hello\\.world" + ], + "result": 1 + }, + { + "id": "sef_3_02", + "source": "sqlite-ext-functions test/regexp.sql 3_02", + "args": [ + "abc123", + "[0-9]+" + ], + "result": 1 + }, + { + "id": "sef_3_03", + "source": "sqlite-ext-functions test/regexp.sql 3_03", + "args": [ + "abc", + "[0-9]+" + ], + "result": 0 + }, + { + "id": "sef_10_01", + "source": "sqlite-ext-functions test/regexp.sql 10_01", + "args": [ + "abracadabra", + "abra(cad)?abra" + ], + "result": 1, + "retiredCResult": 0, + "note": "tiny-regex-c (the retired C engine) did not support groups: '(' and ')' were literal characters, so this pattern could not match and the C suite recorded the opposite outcome (see retiredCResult). Go RE2 supports groups; expectation deliberately updated per DIVERGENCES.md." + }, + { + "id": "sef_11_01", + "source": "sqlite-ext-functions test/regexp.sql 11_01", + "args": [ + "hello", + "" + ], + "result": 1 + }, + { + "id": "sef_12_01", + "source": "sqlite-ext-functions test/regexp.sql 12_01", + "args": [ + "hello", + "h.llo" + ], + "result": 1 + }, + { + "id": "sef_12_02", + "source": "sqlite-ext-functions test/regexp.sql 12_02", + "args": [ + "123abc", + "\\d+\\w+" + ], + "result": 1 + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "abc123", + "\\d+" + ], + "result": 1 + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_replace.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_replace.json new file mode 100644 index 000000000..511e2f8a9 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_replace.json @@ -0,0 +1,119 @@ +{ + "function": "regexp_replace", + "vectors": [ + { + "id": "sef_7_01", + "source": "sqlite-ext-functions test/regexp.sql 7_01", + "args": [ + null, + "pattern", + "replacement" + ], + "result": null + }, + { + "id": "sef_7_02", + "source": "sqlite-ext-functions test/regexp.sql 7_02", + "args": [ + "source", + null, + "replacement" + ], + "result": null + }, + { + "id": "sef_7_03", + "source": "sqlite-ext-functions test/regexp.sql 7_03", + "args": [ + "source", + "pattern", + null + ], + "result": null + }, + { + "id": "sef_8_01", + "source": "sqlite-ext-functions test/regexp.sql 8_01", + "args": [ + "hello world", + "world", + "SQLite" + ], + "result": "hello SQLite" + }, + { + "id": "sef_8_02", + "source": "sqlite-ext-functions test/regexp.sql 8_02", + "args": [ + "123-456-7890", + "[0-9]", + "X" + ], + "result": "XXX-XXX-XXXX" + }, + { + "id": "sef_9_01", + "source": "sqlite-ext-functions test/regexp.sql 9_01", + "args": [ + "file.txt", + "\\.txt$", + ".bak" + ], + "result": "file.bak" + }, + { + "id": "sef_9_02", + "source": "sqlite-ext-functions test/regexp.sql 9_02", + "args": [ + "abc123xyz", + "[0-9]+", + "456" + ], + "result": "abc456xyz" + }, + { + "id": "sef_10_02", + "source": "sqlite-ext-functions test/regexp.sql 10_02", + "args": [ + "abracadabra", + "abra(cad)?abra", + "magic" + ], + "result": "magic", + "retiredCResult": "abracadabra", + "note": "tiny-regex-c (the retired C engine) did not support groups: '(' and ')' were literal characters, so this pattern could not match and the C suite recorded the opposite outcome (see retiredCResult). Go RE2 supports groups; expectation deliberately updated per DIVERGENCES.md." + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "abc123", + "\\d", + "X" + ], + "result": "abcXXX" + }, + { + "id": "cref_no_match", + "source": "C reference implementation behavior (regexp.c)", + "args": [ + "hello", + "xyz", + "Q" + ], + "result": "hello", + "note": "no match returns the source unchanged per the C implementation" + }, + { + "id": "cref_literal_replacement", + "source": "C reference implementation behavior (regexp.c)", + "args": [ + "ab12cd", + "([a-z]+)", + "<$1>" + ], + "result": "<$1>12<$1>", + "note": "the C implementation copied the replacement text verbatim; $1 is not a capture token" + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_substr.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_substr.json new file mode 100644 index 000000000..60188ab82 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/regexp_substr.json @@ -0,0 +1,105 @@ +{ + "function": "regexp_substr", + "vectors": [ + { + "id": "sef_4_01", + "source": "sqlite-ext-functions test/regexp.sql 4_01", + "args": [ + null, + "pattern" + ], + "result": null + }, + { + "id": "sef_4_02", + "source": "sqlite-ext-functions test/regexp.sql 4_02", + "args": [ + "source", + null + ], + "result": null + }, + { + "id": "sef_5_01", + "source": "sqlite-ext-functions test/regexp.sql 5_01", + "args": [ + "hello world", + "w.*d" + ], + "result": "world" + }, + { + "id": "sef_5_02", + "source": "sqlite-ext-functions test/regexp.sql 5_02", + "args": [ + "123-456-7890", + "[0-9]+-[0-9]+-[0-9]+" + ], + "result": "123-456-7890" + }, + { + "id": "sef_6_01", + "source": "sqlite-ext-functions test/regexp.sql 6_01", + "args": [ + "file.txt", + "\\.\\w+$" + ], + "result": ".txt" + }, + { + "id": "sef_6_02", + "source": "sqlite-ext-functions test/regexp.sql 6_02", + "args": [ + "abc123xyz", + "[0-9]+" + ], + "result": "123" + }, + { + "id": "sef_11_02", + "source": "sqlite-ext-functions test/regexp.sql 11_02", + "args": [ + "hello", + "" + ], + "result": "" + }, + { + "id": "sef_12_03", + "source": "sqlite-ext-functions test/regexp.sql 12_03", + "args": [ + "hello world", + "^hello" + ], + "result": "hello" + }, + { + "id": "sef_12_04", + "source": "sqlite-ext-functions test/regexp.sql 12_04", + "args": [ + "end of line", + "line$" + ], + "result": "line" + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "abc123def", + "\\d+" + ], + "result": "123" + }, + { + "id": "cref_no_match", + "source": "C reference implementation behavior (regexp.c)", + "args": [ + "abc", + "[0-9]" + ], + "result": null, + "note": "no match yields NULL per the C implementation" + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/testdata/split_part.json b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/split_part.json new file mode 100644 index 000000000..d2f84f711 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/testdata/split_part.json @@ -0,0 +1,306 @@ +{ + "function": "split_part", + "vectors": [ + { + "id": "sef_1_01", + "source": "sqlite-ext-functions test/split_part.sql 1_01", + "args": [ + null, + "/", + 1 + ], + "result": null + }, + { + "id": "sef_1_02", + "source": "sqlite-ext-functions test/split_part.sql 1_02", + "args": [ + "a/b/c", + null, + 1 + ], + "result": null + }, + { + "id": "sef_1_03", + "source": "sqlite-ext-functions test/split_part.sql 1_03", + "args": [ + "a/b/c", + "/", + null + ], + "result": null + }, + { + "id": "sef_2_01", + "source": "sqlite-ext-functions test/split_part.sql 2_01", + "args": [ + "a/b/c", + "/", + 1 + ], + "result": "a" + }, + { + "id": "sef_2_02", + "source": "sqlite-ext-functions test/split_part.sql 2_02", + "args": [ + "a/b/c", + "/", + 2 + ], + "result": "b" + }, + { + "id": "sef_2_03", + "source": "sqlite-ext-functions test/split_part.sql 2_03", + "args": [ + "a/b/c", + "/", + 3 + ], + "result": "c" + }, + { + "id": "sef_3_01", + "source": "sqlite-ext-functions test/split_part.sql 3_01", + "args": [ + "/a/b/c/", + "/", + 1 + ], + "result": "" + }, + { + "id": "sef_3_02", + "source": "sqlite-ext-functions test/split_part.sql 3_02", + "args": [ + "/a/b/c/", + "/", + 2 + ], + "result": "a" + }, + { + "id": "sef_3_03", + "source": "sqlite-ext-functions test/split_part.sql 3_03", + "args": [ + "/a/b/c/", + "/", + 4 + ], + "result": "c" + }, + { + "id": "sef_3_04", + "source": "sqlite-ext-functions test/split_part.sql 3_04", + "args": [ + "/a/b/c/", + "/", + 5 + ], + "result": "" + }, + { + "id": "sef_4_01", + "source": "sqlite-ext-functions test/split_part.sql 4_01", + "args": [ + "a/b/c", + "/", + -1 + ], + "result": "c" + }, + { + "id": "sef_4_02", + "source": "sqlite-ext-functions test/split_part.sql 4_02", + "args": [ + "a/b/c", + "/", + -2 + ], + "result": "b" + }, + { + "id": "sef_4_03", + "source": "sqlite-ext-functions test/split_part.sql 4_03", + "args": [ + "a/b/c", + "/", + -3 + ], + "result": "a" + }, + { + "id": "sef_5_01", + "source": "sqlite-ext-functions test/split_part.sql 5_01", + "args": [ + "a//b/c", + "/", + 1 + ], + "result": "a" + }, + { + "id": "sef_5_02", + "source": "sqlite-ext-functions test/split_part.sql 5_02", + "args": [ + "a//b/c", + "/", + 2 + ], + "result": "" + }, + { + "id": "sef_5_03", + "source": "sqlite-ext-functions test/split_part.sql 5_03", + "args": [ + "a//b/c", + "/", + 3 + ], + "result": "b" + }, + { + "id": "sef_6_01", + "source": "sqlite-ext-functions test/split_part.sql 6_01", + "args": [ + "a/b/c", + "/", + 4 + ], + "result": null + }, + { + "id": "sef_6_02", + "source": "sqlite-ext-functions test/split_part.sql 6_02", + "args": [ + "a/b/c", + "/", + -4 + ], + "result": null + }, + { + "id": "sef_7_01", + "source": "sqlite-ext-functions test/split_part.sql 7_01", + "args": [ + "a-b-c", + "-", + 2 + ], + "result": "b" + }, + { + "id": "sef_7_02", + "source": "sqlite-ext-functions test/split_part.sql 7_02", + "args": [ + "a#b#c", + "#", + 3 + ], + "result": "c" + }, + { + "id": "sef_8_01", + "source": "sqlite-ext-functions test/split_part.sql 8_01", + "args": [ + "a$b$c", + "$", + 2 + ], + "result": "b" + }, + { + "id": "sef_8_02", + "source": "sqlite-ext-functions test/split_part.sql 8_02", + "args": [ + "a.b.c", + ".", + 3 + ], + "result": "c" + }, + { + "id": "sef_9_01", + "source": "sqlite-ext-functions test/split_part.sql 9_01", + "args": [ + "", + "/", + 1 + ], + "result": "" + }, + { + "id": "sef_9_02", + "source": "sqlite-ext-functions test/split_part.sql 9_02", + "args": [ + "", + "/", + -1 + ], + "result": "" + }, + { + "id": "sef_10_01", + "source": "sqlite-ext-functions test/split_part.sql 10_01", + "args": [ + "a", + "/", + 1 + ], + "result": "a" + }, + { + "id": "sef_10_02", + "source": "sqlite-ext-functions test/split_part.sql 10_02", + "args": [ + "a", + "/", + 2 + ], + "result": null + }, + { + "id": "sef_10_03", + "source": "sqlite-ext-functions test/split_part.sql 10_03", + "args": [ + "a", + "/", + -1 + ], + "result": "a" + }, + { + "id": "pr783_01", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "a,b,c", + ",", + 2 + ], + "result": "b" + }, + { + "id": "pr783_02", + "source": "stackql PR #783 cgo golden sample", + "args": [ + "a,b,c", + ",", + -1 + ], + "result": "c" + }, + { + "id": "cref_empty_sep", + "source": "C reference implementation behavior (split_part.c)", + "args": [ + "a/b/c", + "", + 1 + ], + "result": null, + "note": "empty separator yields NULL per the C guard (*sep == '\\0')" + } + ] +} diff --git a/internal/stackql/sqlengine/internal/sqlfuncs/vectors_test.go b/internal/stackql/sqlengine/internal/sqlfuncs/vectors_test.go new file mode 100644 index 000000000..09129ed89 --- /dev/null +++ b/internal/stackql/sqlengine/internal/sqlfuncs/vectors_test.go @@ -0,0 +1,232 @@ +package sqlfuncs //nolint:testpackage // Exercises private scalar conversion helpers. + +import ( + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "testing" +) + +// goldenVector is one recorded input/output pair. Vectors are sourced from +// the sqlite-ext-functions test suite, the cgo golden samples in stackql +// PR #783, and the retired C reference implementations, as recorded in each +// vector's source field. +type goldenVector struct { + ID string `json:"id"` + Source string `json:"source"` + Args []any `json:"args"` + Result any `json:"result"` + Error bool `json:"error"` + RetiredCResult any `json:"retiredCResult"` + Note string `json:"note"` +} + +type goldenFile struct { + Function string `json:"function"` + Vectors []goldenVector `json:"vectors"` +} + +// normalizeVectorValue converts JSON-decoded values to the types the +// functions produce: integral float64 becomes int64 (SQLite INTEGER), all +// other kinds pass through. +func normalizeVectorValue(v any) any { + if f, ok := v.(float64); ok { + if math.Trunc(f) == f && !math.IsInf(f, 0) { + return int64(f) + } + } + return v +} + +func loadGoldenVectors(t *testing.T, name string) []goldenVector { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", name+".json")) + if err != nil { + t.Fatalf("cannot read golden vectors for %s: %v", name, err) + } + var doc goldenFile + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("cannot parse golden vectors for %s: %v", name, err) + } + if doc.Function != name { + t.Fatalf("golden file %s declares function %q", name, doc.Function) + } + seen := map[string]struct{}{} + for i := range doc.Vectors { + vec := &doc.Vectors[i] + if vec.ID == "" || vec.Source == "" { + t.Fatalf("golden vector %d in %s lacks id or source", i, name) + } + if _, dup := seen[vec.ID]; dup { + t.Fatalf("duplicate golden vector id %q in %s", vec.ID, name) + } + seen[vec.ID] = struct{}{} + for j := range vec.Args { + vec.Args[j] = normalizeVectorValue(vec.Args[j]) + } + vec.Result = normalizeVectorValue(vec.Result) + } + return doc.Vectors +} + +// runGoldenVectors drives fn over every vector and asserts the recorded +// outcome. +func runGoldenVectors(t *testing.T, name string, fn func(args []any) (any, error)) { + t.Helper() + vectors := loadGoldenVectors(t, name) + if len(vectors) == 0 { + t.Fatalf("no golden vectors for %s", name) + } + for _, vec := range vectors { + vec := vec + t.Run(vec.ID, func(t *testing.T) { + got, err := fn(vec.Args) + if vec.Error { + if err == nil { + t.Fatalf("%s(%s): expected an error, got %v", name, formatArgs(vec.Args), got) + } + return + } + if err != nil { + t.Fatalf("%s(%s): unexpected error: %v", name, formatArgs(vec.Args), err) + } + if got != vec.Result { + t.Fatalf("%s(%s) = %#v, want %#v (source: %s)", name, formatArgs(vec.Args), got, vec.Result, vec.Source) + } + }) + } +} + +func formatArgs(args []any) string { + out := "" + for i, a := range args { + if i > 0 { + out += ", " + } + out += fmt.Sprintf("%#v", a) + } + return out +} + +func TestSplitPartGoldenVectors(t *testing.T) { + runGoldenVectors(t, "split_part", func(args []any) (any, error) { + return SplitPart(args[0], args[1], args[2]) + }) +} + +func TestRegexpLikeGoldenVectors(t *testing.T) { + runGoldenVectors(t, "regexp_like", func(args []any) (any, error) { + return RegexpLike(args[0], args[1]) + }) +} + +func TestRegexpGoldenVectors(t *testing.T) { + runGoldenVectors(t, "regexp", func(args []any) (any, error) { + return Regexp(args[0], args[1]) + }) +} + +func TestRegexpSubstrGoldenVectors(t *testing.T) { + runGoldenVectors(t, "regexp_substr", func(args []any) (any, error) { + return RegexpSubstr(args[0], args[1]) + }) +} + +func TestRegexpReplaceGoldenVectors(t *testing.T) { + runGoldenVectors(t, "regexp_replace", func(args []any) (any, error) { + return RegexpReplace(args[0], args[1], args[2]) + }) +} + +func TestJSONEqualGoldenVectors(t *testing.T) { + runGoldenVectors(t, "json_equal", func(args []any) (any, error) { + return JSONEqual(args[0], args[1]) + }) +} + +func TestAWSPolicyEqualGoldenVectors(t *testing.T) { + runGoldenVectors(t, "aws_policy_equal", func(args []any) (any, error) { + return AWSPolicyEqual(args[0], args[1]) + }) +} + +// TestRegexpInvalidPattern pins the error contract for unparseable patterns, +// mirroring the "Invalid regular expression" error in the retired C code. +func TestRegexpInvalidPattern(t *testing.T) { + if _, err := RegexpLike("abc", "["); err == nil { + t.Fatal("regexp_like with an invalid pattern must error") + } + if _, err := RegexpSubstr("abc", "["); err == nil { + t.Fatal("regexp_substr with an invalid pattern must error") + } + if _, err := RegexpReplace("abc", "[", "x"); err == nil { + t.Fatal("regexp_replace with an invalid pattern must error") + } +} + +// TestRegexpDotMatchesNewline pins the RE_DOT_MATCHES_NEWLINE=1 behavior of +// the retired tiny-regex-c build: '.' matches any byte including newline. +func TestRegexpDotMatchesNewline(t *testing.T) { + got, err := RegexpLike("a\nb", "a.b") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != int64(1) { + t.Fatalf("dot must match newline (RE_DOT_MATCHES_NEWLINE parity), got %#v", got) + } + sub, err := RegexpSubstr("a\nb", "a.b") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sub != "a\nb" { + t.Fatalf("regexp_substr dot-newline parity broken, got %#v", sub) + } +} + +// TestValueTextCoercion pins the sqlite3_value_text-style coercions applied +// to non-text arguments. +func TestValueTextCoercion(t *testing.T) { + cases := []struct { + in any + want string + wantOK bool + }{ + {nil, "", false}, + {"abc", "abc", true}, + {[]byte("xyz"), "xyz", true}, + {int64(42), "42", true}, + {float64(2.5), "2.5", true}, + {float64(3), "3.0", true}, + } + for _, c := range cases { + got, ok := valueText(c.in) + if got != c.want || ok != c.wantOK { + t.Fatalf("valueText(%#v) = (%q, %v), want (%q, %v)", c.in, got, ok, c.want, c.wantOK) + } + } +} + +// TestValueIntCoercion pins the sqlite3_value_int-style coercions applied to +// the part index of split_part. +func TestValueIntCoercion(t *testing.T) { + cases := []struct { + in any + want int64 + }{ + {nil, 0}, + {int64(-3), -3}, + {float64(2.9), 2}, + {float64(-2.9), -2}, + {"11", 11}, + {" -4abc", -4}, + {"abc", 0}, + {[]byte("7x"), 7}, + } + for _, c := range cases { + if got := valueInt(c.in); got != c.want { + t.Fatalf("valueInt(%#v) = %d, want %d", c.in, got, c.want) + } + } +} diff --git a/internal/stackql/sqlengine/postgres_tcp.go b/internal/stackql/sqlengine/postgres_tcp.go new file mode 100644 index 000000000..7cf653fa5 --- /dev/null +++ b/internal/stackql/sqlengine/postgres_tcp.go @@ -0,0 +1,282 @@ +//nolint:dupl,nolintlint //TODO: fix this +package sqlengine + +import ( + "database/sql" + "fmt" + "os" + "regexp" + "sync" + + "github.com/stackql/stackql/internal/stackql/logging" + db_util "github.com/stackql/stackql/internal/stackql/sqlengine/connect" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" + + _ "github.com/jackc/pgx/v5/stdlib" //nolint:revive,nolintlint // anonymous import is a pattern for SQL drivers +) + +var ( + _ SQLEngine = &postgresTCPEngine{} + prepStmtRegex *regexp.Regexp = regexp.MustCompile(`\$\d+`) //nolint:revive // preferred idiom +) + +type postgresTCPEngine struct { + db *sql.DB + dsn string + controlAttributes sqlcontrol.ControlAttributes + ctrlMutex *sync.Mutex + sessionMutex *sync.Mutex + discoveryMutex *sync.Mutex +} + +func (se *postgresTCPEngine) IsMemory() bool { + return false +} + +func (se *postgresTCPEngine) GetDB() (*sql.DB, error) { + return se.db, nil +} + +func (se *postgresTCPEngine) GetTx() (*sql.Tx, error) { + return se.db.Begin() +} + +func newPostgresTCPEngine( + cfg Config, + controlAttributes sqlcontrol.ControlAttributes, +) (*postgresTCPEngine, error) { + dsn := cfg.DSN + if dsn == "" { + return nil, fmt.Errorf("cannot init postgres from empty dsn") + } + db, err := db_util.Open("pgx", "postgres", cfg.DSN, cfg.InitMaxRetries, cfg.InitRetryInitialDelay) + if err != nil { + return nil, err + } + db.SetConnMaxLifetime(-1) + eng := &postgresTCPEngine{ + db: db, + dsn: dsn, + controlAttributes: controlAttributes, + ctrlMutex: &sync.Mutex{}, + sessionMutex: &sync.Mutex{}, + discoveryMutex: &sync.Mutex{}, + } + if cfg.DbInitFilePath != "" { + err = eng.execFile(cfg.DbInitFilePath) + } + if err != nil { + return eng, err + } + return eng, err +} + +func (se *postgresTCPEngine) execFile(fileName string) error { + fileContents, err := os.ReadFile(fileName) + if err != nil { + return err + } + _, err = se.db.Exec(string(fileContents)) + if err != nil { + return fmt.Errorf("stackql postgres db exec file error: %w", err) + } + return nil +} + +func (se *postgresTCPEngine) execFileLocal(fileName string) error { + expF, err := getFilePathFromRepositoryRoot(fileName) + if err != nil { + return err + } + return se.execFile(expF) +} + +func (se *postgresTCPEngine) ExecFileLocal(fileName string) error { + return se.execFileLocal(fileName) +} + +func (se *postgresTCPEngine) ExecFile(fileName string) error { + return se.execFile(fileName) +} + +func (se postgresTCPEngine) Exec(query string, varArgs ...interface{}) (sql.Result, error) { + res, err := se.db.Exec(query, varArgs...) + return res, err +} + +func (se postgresTCPEngine) QueryRow(query string, varArgs ...interface{}) *sql.Row { + res := se.db.QueryRow(query, varArgs...) + return res +} + +func (se postgresTCPEngine) ExecInTxn(queries []string) error { + txn, err := se.db.Begin() + if err != nil { + return err + } + for _, query := range queries { + _, err = txn.Exec(query) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + } + err = txn.Commit() + return err +} + +func (se postgresTCPEngine) GetNextGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getNextGenerationID() +} + +func (se postgresTCPEngine) GetCurrentGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getCurrentGenerationID() +} + +func (se postgresTCPEngine) GetNextDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getNextProviderGenerationID(discoveryName) +} + +func (se postgresTCPEngine) GetCurrentDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getCurrentProviderGenerationID(discoveryName) +} + +func (se postgresTCPEngine) GetNextSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getNextSessionID(generationID) +} + +func (se postgresTCPEngine) GetCurrentSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getCurrentSessionID(generationID) +} + +func (se postgresTCPEngine) getCurrentGenerationID() (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_generation_id FROM "__iql__.control.generation" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.generation" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm WHERE lhs.collected_dttm IS null`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) getNextGenerationID() (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.generation" (generation_description, created_dttm) VALUES ('', current_timestamp) RETURNING iql_generation_id`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) getCurrentProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_discovery_generation_id FROM "__iql__.control.discovery_generation" lhs INNER JOIN (SELECT discovery_name, max(created_dttm) AS max_dttm FROM "__iql__.control.discovery_generation" WHERE collected_dttm IS null GROUP BY discovery_name) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.discovery_name = rhs.discovery_name WHERE lhs.collected_dttm IS null AND lhs.discovery_name = $1`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) getNextProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.discovery_generation" (discovery_name, created_dttm) VALUES ($1, current_timestamp) RETURNING iql_discovery_generation_id`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) getCurrentSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_session_id FROM "__iql__.control.session" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.session" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.iql_genration_id = rhs.iql_generation_id WHERE lhs.iql_generation_id = $1 AND lhs.collected_dttm IS null`, generationID) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) getNextSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.session" (iql_generation_id, created_dttm) VALUES ($1, current_timestamp) RETURNING iql_session_id`, generationID) + err := res.Scan(&retVal) + logging.GetLogger().Infoln( + fmt.Sprintf( + "getNextSessionID(): generation id = %d, session id = %d", + generationID, + retVal, + ), + ) + return retVal, err +} + +func (se postgresTCPEngine) CacheStoreGet(key string) ([]byte, error) { + var retVal []byte + res := se.db.QueryRow(`SELECT v FROM "__iql__.cache.key_val" WHERE k = $1`, key) + err := res.Scan(&retVal) + return retVal, err +} + +func (se postgresTCPEngine) CacheStorePut(key string, val []byte, tablespace string, tablespaceID int) error { + txn, err := se.db.Begin() + if err != nil { + return err + } + _, err = txn.Exec(`DELETE FROM "__iql__.cache.key_val" WHERE k = $1`, key) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + _, err = txn.Exec( + `INSERT INTO "__iql__.cache.key_val" (k, v, tablespace, tablespace_id) VALUES($1, $2, $3, $4)`, + key, + val, + tablespace, + tablespaceID, + ) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + err = txn.Commit() + return err +} + +// TODO: fix upstream numbering system and remove this +func (se postgresTCPEngine) bodgePrepStmt(s string) (string, bool) { + matches := prepStmtRegex.FindAllStringIndex(s, -1) + if len(matches) == 0 { + return s, false + } + prev := 0 + rv := "" + for i, j := range matches { + rv = rv + s[prev:j[0]] + fmt.Sprintf("$%d", i+1) + prev = j[1] + } + rv += s[matches[len(matches)-1][1]:] + return rv, len(matches) > 0 +} + +func (se postgresTCPEngine) Query(query string, varArgs ...interface{}) (*sql.Rows, error) { + return se.query(query, varArgs...) +} + +func (se postgresTCPEngine) query(query string, varArgs ...interface{}) (*sql.Rows, error) { + bodgedQuery, containsParameters := se.bodgePrepStmt(query) + if !containsParameters { + res, err := se.db.Query(bodgedQuery, []interface{}{}...) + return res, err + } + res, err := se.db.Query(bodgedQuery, varArgs...) + return res, err +} diff --git a/internal/stackql/sqlengine/setup_embed.go b/internal/stackql/sqlengine/setup_embed.go new file mode 100644 index 000000000..603346088 --- /dev/null +++ b/internal/stackql/sqlengine/setup_embed.go @@ -0,0 +1,23 @@ +package sqlengine + +import ( + _ "embed" + "fmt" +) + +//go:embed sql/sqlite/sqlengine-setup.ddl +var sqLiteEngineSetupDDL string + +//go:embed sql/postgres/sqlengine-setup.ddl +var postgresEngineSetupDDL string + +func GetSQLEngineSetupDDL(engineType string) (string, error) { + switch engineType { + case "sqlite": + return sqLiteEngineSetupDDL, nil + case "postgres": + return postgresEngineSetupDDL, nil + default: + return "", fmt.Errorf("unsupported engine type: %s", engineType) + } +} diff --git a/internal/stackql/sqlengine/snowflake_tcp.go b/internal/stackql/sqlengine/snowflake_tcp.go new file mode 100644 index 000000000..0cff4f26b --- /dev/null +++ b/internal/stackql/sqlengine/snowflake_tcp.go @@ -0,0 +1,259 @@ +//nolint:dupl,nolintlint //TODO: fix this +package sqlengine + +import ( + "database/sql" + "fmt" + "os" + "sync" + + "github.com/stackql/stackql/internal/stackql/logging" + db_util "github.com/stackql/stackql/internal/stackql/sqlengine/connect" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" + + _ "github.com/snowflakedb/gosnowflake" //nolint:revive,nolintlint // anonymous import is a pattern for SQL drivers +) + +var ( + _ SQLEngine = &snowflakeTCPEngine{} +) + +type snowflakeTCPEngine struct { + db *sql.DB + dsn string + controlAttributes sqlcontrol.ControlAttributes + ctrlMutex *sync.Mutex + sessionMutex *sync.Mutex + discoveryMutex *sync.Mutex +} + +func (se *snowflakeTCPEngine) IsMemory() bool { + return false +} + +func (se *snowflakeTCPEngine) GetDB() (*sql.DB, error) { + return se.db, nil +} + +func (se *snowflakeTCPEngine) GetTx() (*sql.Tx, error) { + return se.db.Begin() +} + +func newSnowflakeTCPEngine( + cfg Config, + controlAttributes sqlcontrol.ControlAttributes, +) (*snowflakeTCPEngine, error) { + dsn := cfg.DSN + if dsn == "" { + return nil, fmt.Errorf("cannot init snowflake from empty dsn") + } + db, err := db_util.Open("snowflake", "snowflake", cfg.DSN, cfg.InitMaxRetries, cfg.InitRetryInitialDelay) + if err != nil { + return nil, err + } + db.SetConnMaxLifetime(-1) + eng := &snowflakeTCPEngine{ + db: db, + dsn: dsn, + controlAttributes: controlAttributes, + ctrlMutex: &sync.Mutex{}, + sessionMutex: &sync.Mutex{}, + discoveryMutex: &sync.Mutex{}, + } + if cfg.DbInitFilePath != "" { + err = eng.execFile(cfg.DbInitFilePath) + } + if err != nil { + return eng, err + } + return eng, err +} + +func (se *snowflakeTCPEngine) execFile(fileName string) error { + fileContents, err := os.ReadFile(fileName) + if err != nil { + return err + } + _, err = se.db.Exec(string(fileContents)) + if err != nil { + return fmt.Errorf("stackql snowflake db exec file error: %w", err) + } + return nil +} + +func (se *snowflakeTCPEngine) execFileLocal(fileName string) error { + expF, err := getFilePathFromRepositoryRoot(fileName) + if err != nil { + return err + } + return se.execFile(expF) +} + +func (se *snowflakeTCPEngine) ExecFileLocal(fileName string) error { + return se.execFileLocal(fileName) +} + +func (se *snowflakeTCPEngine) ExecFile(fileName string) error { + return se.execFile(fileName) +} + +func (se snowflakeTCPEngine) Exec(query string, varArgs ...interface{}) (sql.Result, error) { + res, err := se.db.Exec(query, varArgs...) + return res, err +} + +func (se snowflakeTCPEngine) QueryRow(query string, varArgs ...interface{}) *sql.Row { + res := se.db.QueryRow(query, varArgs...) + return res +} + +func (se snowflakeTCPEngine) ExecInTxn(queries []string) error { + txn, err := se.db.Begin() + if err != nil { + return err + } + for _, query := range queries { + _, err = txn.Exec(query) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + } + err = txn.Commit() + return err +} + +func (se snowflakeTCPEngine) GetNextGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getNextGenerationID() +} + +func (se snowflakeTCPEngine) GetCurrentGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getCurrentGenerationID() +} + +func (se snowflakeTCPEngine) GetNextDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getNextProviderGenerationID(discoveryName) +} + +func (se snowflakeTCPEngine) GetCurrentDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getCurrentProviderGenerationID(discoveryName) +} + +func (se snowflakeTCPEngine) GetNextSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getNextSessionID(generationID) +} + +func (se snowflakeTCPEngine) GetCurrentSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getCurrentSessionID(generationID) +} + +func (se snowflakeTCPEngine) getCurrentGenerationID() (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_generation_id FROM "__iql__.control.generation" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.generation" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm WHERE lhs.collected_dttm IS null`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) getNextGenerationID() (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.generation" (generation_description, created_dttm) VALUES ('', current_timestamp) RETURNING iql_generation_id`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) getCurrentProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_discovery_generation_id FROM "__iql__.control.discovery_generation" lhs INNER JOIN (SELECT discovery_name, max(created_dttm) AS max_dttm FROM "__iql__.control.discovery_generation" WHERE collected_dttm IS null GROUP BY discovery_name) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.discovery_name = rhs.discovery_name WHERE lhs.collected_dttm IS null AND lhs.discovery_name = $1`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) getNextProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.discovery_generation" (discovery_name, created_dttm) VALUES ($1, current_timestamp) RETURNING iql_discovery_generation_id`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) getCurrentSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_session_id FROM "__iql__.control.session" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.session" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.iql_genration_id = rhs.iql_generation_id WHERE lhs.iql_generation_id = $1 AND lhs.collected_dttm IS null`, generationID) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) getNextSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.session" (iql_generation_id, created_dttm) VALUES ($1, current_timestamp) RETURNING iql_session_id`, generationID) + err := res.Scan(&retVal) + logging.GetLogger().Infoln( + fmt.Sprintf( + "getNextSessionID(): generation id = %d, session id = %d", + generationID, + retVal, + ), + ) + return retVal, err +} + +func (se snowflakeTCPEngine) CacheStoreGet(key string) ([]byte, error) { + var retVal []byte + res := se.db.QueryRow(`SELECT v FROM "__iql__.cache.key_val" WHERE k = $1`, key) + err := res.Scan(&retVal) + return retVal, err +} + +func (se snowflakeTCPEngine) CacheStorePut(key string, val []byte, tablespace string, tablespaceID int) error { + txn, err := se.db.Begin() + if err != nil { + return err + } + _, err = txn.Exec(`DELETE FROM "__iql__.cache.key_val" WHERE k = $1`, key) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + _, err = txn.Exec( + `INSERT INTO "__iql__.cache.key_val" (k, v, tablespace, tablespace_id) VALUES($1, $2, $3, $4)`, + key, + val, + tablespace, + tablespaceID, + ) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + err = txn.Commit() + return err +} + +func (se snowflakeTCPEngine) Query(query string, varArgs ...interface{}) (*sql.Rows, error) { + return se.query(query, varArgs...) +} + +func (se snowflakeTCPEngine) query(query string, varArgs ...interface{}) (*sql.Rows, error) { + res, err := se.db.Query(query, varArgs...) + return res, err +} diff --git a/internal/stackql/sqlengine/sql/postgres/sqlengine-setup.ddl b/internal/stackql/sqlengine/sql/postgres/sqlengine-setup.ddl new file mode 100644 index 000000000..893dc4671 --- /dev/null +++ b/internal/stackql/sqlengine/sql/postgres/sqlengine-setup.ddl @@ -0,0 +1,496 @@ + +CREATE TABLE IF NOT EXISTS "__iql__.control.generation" ( + iql_generation_id BIGSERIAL PRIMARY KEY + ,generation_description TEXT + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,collected_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.generation.created_dttm" +ON "__iql__.control.generation" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.discovery_generation" ( + iql_discovery_generation_id BIGSERIAL PRIMARY KEY + ,discovery_name TEXT NOT NULL + ,discovery_generation_description TEXT + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,collected_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.discovery_generation.created_dttm" +ON "__iql__.control.discovery_generation" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.session" ( + iql_session_id BIGSERIAL PRIMARY KEY + ,iql_generation_id INTEGER NOT NULL + ,session_description TEXT + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,collected_dttm TIMESTAMP WITH TIME ZONE DEFAULT null + ,FOREIGN KEY(iql_generation_id) REFERENCES "__iql__.control.generation"(iql_generation_id) +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.session.created_dttm" +ON "__iql__.control.session" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.cache.key_val" ( + k TEXT NOT NULL UNIQUE + ,v BYTEA + ,tablespace TEXT + ,tablespace_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "__iql__.control.gc.txn_table_x_ref" ( + iql_generation_id INTEGER not null + ,iql_session_id INTEGER not null + ,iql_transaction_id INTEGER not null + ,table_name TEXT not null + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,collected_dttm TIMESTAMP WITH TIME ZONE DEFAULT null + ,PRIMARY KEY (iql_generation_id, iql_session_id, iql_transaction_id, table_name) +) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.gc.rings" ( + ring_id BIGSERIAL PRIMARY KEY + ,ring_name TEXT not null UNIQUE + ,current_value INTEGER not null DEFAULT 0 + ,current_offset INTEGER not null DEFAULT 0 + ,width_bits INTEGER not null DEFAULT 32 + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,collected_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.gc.rings.ring_name" +ON "__iql__.control.gc.rings" (ring_name) +; + +INSERT INTO "__iql__.control.gc.rings" (ring_name) +VALUES ('transaction_id') +ON CONFLICT (ring_name) DO NOTHING +; + +INSERT INTO "__iql__.control.gc.rings" (ring_name) +VALUES ('session_id') +ON CONFLICT (ring_name) DO NOTHING +; + + +CREATE TABLE IF NOT EXISTS "__iql__.views" ( + iql_view_id BIGSERIAL PRIMARY KEY + ,view_name TEXT NOT NULL UNIQUE + ,view_ddl TEXT + ,view_stackql_ddl TEXT + ,required_params TEXT NOT NULL DEFAULT '' + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,deleted_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.views" +ON "__iql__.views" (view_name) +; + +INSERT INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'stackql_repositories', + 'select id, name, url, org from github.repos.repos where org = ''stackql'';' +) +ON CONFLICT (view_name) DO NOTHING +; + + +INSERT INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'aws_ec2_all_volumes', + 'select + ''ap-southeast-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-southeast-2'' + UNION + SELECT + ''ap-southeast-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-southeast-1'' + UNION + SELECT + ''ap-northeast-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-1'' + UNION + SELECT + ''ap-northeast-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-2'' + UNION + SELECT + ''ap-northeast-3'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-3'' + UNION + SELECT + ''ap-south-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-south-1'' + UNION + SELECT + ''us-east-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-east-1'' + UNION + SELECT + ''us-east-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-east-2'' + UNION + SELECT + ''us-west-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-west-1'' + UNION + SELECT + ''us-west-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-west-2'' + UNION + SELECT + ''ca-central-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ca-central-1'' + UNION + SELECT + ''sa-east-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''sa-east-1'' + UNION + SELECT + ''eu-central-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-central-1'' + UNION + SELECT + ''eu-north-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-north-1'' + UNION + SELECT + ''eu-west-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-1'' + UNION + SELECT + ''eu-west-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-2'' + UNION + SELECT + ''eu-west-3'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-3'' + ORDER BY size DESC + ;' +) +ON CONFLICT (view_name) DO NOTHING +; + +CREATE TABLE IF NOT EXISTS "__iql__.external.columns" ( + iql_column_id BIGSERIAL PRIMARY KEY + ,connection_name TEXT + ,catalog_name TEXT + ,schema_name TEXT + ,table_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision INT + ,UNIQUE(connection_name, catalog_name, schema_name, table_name, column_name) +) +; + +CREATE TABLE IF NOT EXISTS "__iql__.materialized_views" ( + iql_view_id BIGSERIAL PRIMARY KEY + ,view_name TEXT NOT NULL UNIQUE + ,view_ddl TEXT + ,translated_ddl TEXT + ,translated_inline_dml TEXT -- for systems that do not have materialized views + ,view_stackql_ddl TEXT + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,deleted_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.materialized_views" +ON "__iql__.materialized_views" (view_name) +; + +CREATE TABLE IF NOT EXISTS "__iql__.materialized_views.columns" ( + iql_column_id BIGSERIAL PRIMARY KEY + ,view_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision INT + ,UNIQUE(view_name, column_name) +) +; + +INSERT INTO "__iql__.materialized_views" ( + view_name, + view_ddl, + translated_ddl +) +VALUES ( + 'stackql_gossip', + ' + create materialized view stackql_gossip as + select ''stackql is open to extension'' as gossip, ''tech'' as category + ', + ' + create materialized view stackql_gossip as + select ''stackql is open to extension'' as gossip, ''tech'' as category + ' +) +ON CONFLICT (view_name) DO NOTHING +; + +INSERT INTO "__iql__.materialized_views.columns" ( + view_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_gossip', + 'gossip', + 'text', + 1, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (view_name, column_name) DO NOTHING +; + +INSERT INTO "__iql__.materialized_views.columns" ( + view_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_gossip', + 'category', + 'text', + 2, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (view_name, column_name) DO NOTHING +; + + +create materialized view if not exists stackql_gossip +as +select 'stackql is open to extension' as gossip, 'tech' as category +union all +select 'stackql wants to hear from you' as gossip, 'community' as category +union all +select 'stackql is not opinionated' as gossip, 'opinion' as category +; + +CREATE INDEX IF NOT EXISTS "idx.stackql_gossip_gossip" on stackql_gossip(gossip); + + +CREATE TABLE IF NOT EXISTS "__iql__.tables" ( + iql_table_id BIGSERIAL PRIMARY KEY + ,table_name TEXT NOT NULL UNIQUE + ,table_type TEXT + ,table_ddl TEXT + ,translated_ddl TEXT + ,translated_inline_dml TEXT -- for create like future proofing + ,iql_generation_id INTEGER -- for temp tables + ,iql_session_id INTEGER -- for temp tables + ,iql_txn_id INTEGER -- for temp tables + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,deleted_dttm TIMESTAMP WITH TIME ZONE DEFAULT null + ,UNIQUE(table_name, table_type, iql_generation_id, iql_session_id, iql_txn_id, deleted_dttm) +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.tables" +ON "__iql__.tables" (table_name) +; + +CREATE TABLE IF NOT EXISTS "__iql__.tables.columns" ( + iql_column_id BIGSERIAL PRIMARY KEY + ,table_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision INT + ,UNIQUE(table_name, column_name) +) +; + +INSERT INTO "__iql__.tables" ( + table_name, + table_type, + table_ddl, + translated_ddl, + translated_inline_dml +) +VALUES ( + 'stackql_notes', + null, -- null or empty string for non temp table + ' + create table if not exists stackql_notes( + note_id BIGSERIAL PRIMARY KEY, + note text UNIQUE, + priority int + ) + ; + ', + ' + create table stackql_notes( + note_id BIGSERIAL PRIMARY KEY, + note text UNIQUE, + priority int + ) + ; + ', + NULL +) +ON CONFLICT (table_name) DO NOTHING +; + +INSERT INTO "__iql__.tables.columns" ( + table_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_notes', + 'note', + 'text', + 1, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (table_name, column_name) DO NOTHING +; + +INSERT INTO "__iql__.tables.columns" ( + table_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_notes', + 'priority', + 'int', + 2, + 1700, -- oid for numeric + 0, + 0 +) +ON CONFLICT (table_name, column_name) DO NOTHING +; + +create table if not exists stackql_notes( + note_id BIGSERIAL PRIMARY KEY, + note text UNIQUE, + priority int +) +; + +INSERT INTO stackql_notes(note, priority) +select 'v0.5.418 introduced table valued functions, for example json_each.' as note, 1000 as priority +union all +select 'stackql supports the postgres wire protocol.' as note, 10 as priority +on conflict (note) do nothing +; diff --git a/internal/stackql/sqlengine/sql/sqlite/sqlengine-setup.ddl b/internal/stackql/sqlengine/sql/sqlite/sqlengine-setup.ddl new file mode 100644 index 000000000..4a96bb22d --- /dev/null +++ b/internal/stackql/sqlengine/sql/sqlite/sqlengine-setup.ddl @@ -0,0 +1,544 @@ + +CREATE TABLE IF NOT EXISTS "__iql__.control.generation" ( + iql_generation_id INTEGER PRIMARY KEY AUTOINCREMENT + ,generation_description TEXT + ,created_dttm INTEGER not null + ,collected_dttm INTEGER default null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.generation.created_dttm" +ON "__iql__.control.generation" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.discovery_generation" ( + iql_discovery_generation_id INTEGER PRIMARY KEY AUTOINCREMENT + ,discovery_name TEXT NOT NULL + ,discovery_generation_description TEXT + ,created_dttm INTEGER not null + ,collected_dttm INTEGER default null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.discovery_generation.created_dttm" +ON "__iql__.control.discovery_generation" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.session" ( + iql_session_id INTEGER PRIMARY KEY AUTOINCREMENT + ,iql_generation_id INTEGER NOT NULL + ,session_description TEXT + ,created_dttm INTEGER not null + ,collected_dttm INTEGER default null + ,FOREIGN KEY(iql_generation_id) REFERENCES "__iql__.control.generation"(iql_generation_id) +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.session.created_dttm" +ON "__iql__.control.session" (created_dttm) +; + +CREATE TABLE IF NOT EXISTS "__iql__.cache.key_val" ( + k TEXT NOT NULL UNIQUE + ,v BLOB + ,tablespace TEXT + ,tablespace_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "__iql__.control.gc.txn_table_x_ref" ( + iql_generation_id INTEGER not null + ,iql_session_id INTEGER not null + ,iql_transaction_id INTEGER not null + ,table_name TEXT not null + ,created_dttm not null default CURRENT_TIMESTAMP + ,collected_dttm INTEGER default null + ,PRIMARY KEY (iql_generation_id, iql_session_id, iql_transaction_id, table_name) +) +; + +CREATE TABLE IF NOT EXISTS "__iql__.control.gc.rings" ( + ring_id INTEGER PRIMARY KEY AUTOINCREMENT + ,ring_name TEXT not null UNIQUE + ,current_value INTEGER not null DEFAULT 0 + ,current_offset INTEGER not null DEFAULT 0 + ,width_bits INTEGER not null DEFAULT 32 + ,created_dttm DateTime not null default CURRENT_TIMESTAMP + ,collected_dttm DateTime default null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.control.gc.rings.ring_name" +ON "__iql__.control.gc.rings" (ring_name) +; + +INSERT OR IGNORE INTO "__iql__.control.gc.rings" (ring_name) VALUES ('transaction_id'); + +INSERT OR IGNORE INTO "__iql__.control.gc.rings" (ring_name) VALUES ('session_id'); + +CREATE TABLE IF NOT EXISTS "__iql__.views" ( + iql_view_id INTEGER PRIMARY KEY AUTOINCREMENT + ,view_name TEXT NOT NULL UNIQUE + ,view_ddl TEXT + ,view_stackql_ddl TEXT + ,required_params TEXT NOT NULL DEFAULT '' + ,created_dttm DateTime not null default CURRENT_TIMESTAMP + ,deleted_dttm DateTime DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.views" +ON "__iql__.views" (view_name) +; + +INSERT OR IGNORE INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'stackql_repositories', + 'select id, name, url, org from github.repos.repos where org = ''stackql'';' +) +; + + +INSERT OR IGNORE INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'aws_ec2_all_volumes', + 'select + ''ap-southeast-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-southeast-2'' + UNION + SELECT + ''ap-southeast-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-southeast-1'' + UNION + SELECT + ''ap-northeast-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-1'' + UNION + SELECT + ''ap-northeast-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-2'' + UNION + SELECT + ''ap-northeast-3'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-northeast-3'' + UNION + SELECT + ''ap-south-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ap-south-1'' + UNION + SELECT + ''us-east-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-east-1'' + UNION + SELECT + ''us-east-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-east-2'' + UNION + SELECT + ''us-west-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-west-1'' + UNION + SELECT + ''us-west-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''us-west-2'' + UNION + SELECT + ''ca-central-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''ca-central-1'' + UNION + SELECT + ''sa-east-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''sa-east-1'' + UNION + SELECT + ''eu-central-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-central-1'' + UNION + SELECT + ''eu-north-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-north-1'' + UNION + SELECT + ''eu-west-1'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-1'' + UNION + SELECT + ''eu-west-2'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-2'' + UNION + SELECT + ''eu-west-3'' AS aws_region, + volumeId, + encrypted, + size + from aws.ec2.volumes + where region = ''eu-west-3'' + ORDER BY size DESC + ;' +) +; + +INSERT OR IGNORE INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'aws_cc_bucket_detail', + ' + SELECT + JSON_EXTRACT(Properties, ''$.Arn'') as Arn, + JSON_EXTRACT(Properties, ''$.BucketName'') as BucketName, + JSON_EXTRACT(Properties, ''$.DomainName'') as DomainName, + JSON_EXTRACT(Properties, ''$.RegionalDomainName'') as RegionalDomainName, + JSON_EXTRACT(Properties, ''$.DualStackDomainName'') as DualStackDomainName, + JSON_EXTRACT(Properties, ''$.WebsiteURL'') as WebsiteURL, + JSON_EXTRACT(Properties, ''$.OwnershipControls.Rules[0].ObjectOwnership'') as ObjectOwnership, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.RestrictPublicBuckets'') = 0, ''false'', ''true'') as RestrictPublicBuckets, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicPolicy'') = 0, ''false'', ''true'') as BlockPublicPolicy, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicAcls'') = 0, ''false'', ''true'') as BlockPublicAcls, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.IgnorePublicAcls'') = 0, ''false'', ''true'') as IgnorePublicAcls, + JSON_EXTRACT(Properties, ''$.Tags'') as Tags + FROM aws.cloud_control.resource WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' and data__Identifier = ''stackql-trial-bucket-01'' + ;' +); + +INSERT OR IGNORE INTO "__iql__.views" ( + view_name, + view_ddl +) +VALUES ( + 'aws_cc_bucket_unfiltered', + ' + SELECT + JSON_EXTRACT(Properties, ''$.Arn'') as Arn, + JSON_EXTRACT(Properties, ''$.BucketName'') as BucketName, + JSON_EXTRACT(Properties, ''$.DomainName'') as DomainName, + JSON_EXTRACT(Properties, ''$.RegionalDomainName'') as RegionalDomainName, + JSON_EXTRACT(Properties, ''$.DualStackDomainName'') as DualStackDomainName, + JSON_EXTRACT(Properties, ''$.WebsiteURL'') as WebsiteURL, + JSON_EXTRACT(Properties, ''$.OwnershipControls.Rules[0].ObjectOwnership'') as ObjectOwnership, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.RestrictPublicBuckets'') = 0, ''false'', ''true'') as RestrictPublicBuckets, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicPolicy'') = 0, ''false'', ''true'') as BlockPublicPolicy, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicAcls'') = 0, ''false'', ''true'') as BlockPublicAcls, + IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.IgnorePublicAcls'') = 0, ''false'', ''true'') as IgnorePublicAcls, + JSON_EXTRACT(Properties, ''$.Tags'') as Tags + FROM aws.cloud_control_legacy.resources WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' + ;' +); + +CREATE TABLE IF NOT EXISTS "__iql__.external.columns" ( + iql_column_id INTEGER PRIMARY KEY AUTOINCREMENT + ,connection_name TEXT + ,catalog_name TEXT + ,schema_name TEXT + ,table_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision TEXT + ,UNIQUE(connection_name, catalog_name, schema_name, table_name, column_name) +) +; + +CREATE TABLE IF NOT EXISTS "__iql__.materialized_views" ( + iql_view_id INTEGER PRIMARY KEY AUTOINCREMENT + ,view_name TEXT NOT NULL UNIQUE + ,view_ddl TEXT + ,translated_ddl TEXT + ,translated_inline_dml TEXT -- for systems that do not have materialized views + ,view_stackql_ddl TEXT + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,deleted_dttm TIMESTAMP WITH TIME ZONE DEFAULT null +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.materialized_views" +ON "__iql__.materialized_views" (view_name) +; + +CREATE TABLE IF NOT EXISTS "__iql__.materialized_views.columns" ( + iql_column_id INTEGER PRIMARY KEY AUTOINCREMENT + ,view_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision INT + ,UNIQUE(view_name, column_name) +) +; + +INSERT INTO "__iql__.materialized_views" ( + view_name, + view_ddl, + translated_ddl, + translated_inline_dml +) +VALUES ( + 'stackql_gossip', + ' + create materialized view stackql_gossip as + select ''stackql is open to extension'' as gossip, ''tech'' as category + ', + ' + create table stackql_gossip( + gossip text UNIQUE, + category text) + ', + ' + insert into stackql_gossip(gossip, category) + select ''stackql is open to extension'' as gossip, ''tech'' as category + ' +) +ON CONFLICT (view_name) DO NOTHING +; + +INSERT INTO "__iql__.materialized_views.columns" ( + view_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_gossip', + 'gossip', + 'text', + 1, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (view_name, column_name) DO NOTHING +; + +INSERT INTO "__iql__.materialized_views.columns" ( + view_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_gossip', + 'category', + 'text', + 2, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (view_name, column_name) DO NOTHING +; + +create table if not exists stackql_gossip( + gossip text UNIQUE, + category text) +; + +CREATE INDEX IF NOT EXISTS "idx.stackql_gossip_gossip" on stackql_gossip(category); + +INSERT INTO stackql_gossip(gossip, category) +select 'stackql is open to extension' as gossip, 'tech' as category +union all +select 'stackql wants to hear from you' as gossip, 'community' as category +union all +select 'stackql is not opinionated' as gossip, 'opinion' as category +on conflict (gossip) do nothing +; + +CREATE TABLE IF NOT EXISTS "__iql__.tables" ( + iql_table_id INTEGER PRIMARY KEY AUTOINCREMENT + ,table_name TEXT NOT NULL UNIQUE + ,table_type TEXT + ,table_ddl TEXT + ,translated_ddl TEXT + ,translated_inline_dml TEXT -- for create like future proofing + ,iql_generation_id INTEGER -- for temp tables + ,iql_session_id INTEGER -- for temp tables + ,iql_txn_id INTEGER -- for temp tables + ,created_dttm TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ,deleted_dttm TIMESTAMP WITH TIME ZONE DEFAULT null + ,UNIQUE(table_name, table_type, iql_generation_id, iql_session_id, iql_txn_id, deleted_dttm) +) +; + +CREATE INDEX IF NOT EXISTS "idx.__iql__.tables" +ON "__iql__.tables" (table_name) +; + +CREATE TABLE IF NOT EXISTS "__iql__.tables.columns" ( + iql_column_id INTEGER PRIMARY KEY AUTOINCREMENT + ,table_name TEXT + ,column_name TEXT + ,column_type TEXT + ,ordinal_position INT + ,"oid" INT + ,column_width INT + ,column_precision INT + ,UNIQUE(table_name, column_name) +) +; + +INSERT INTO "__iql__.tables" ( + table_name, + table_type, + table_ddl, + translated_ddl, + translated_inline_dml +) +VALUES ( + 'stackql_notes', + null, -- null or empty string for non temp table + ' + create table stackql_notes( + note_id INTEGER PRIMARY KEY AUTOINCREMENT, + note text UNIQUE, + priority int + ) + ; + ', + ' + create table stackql_notes( + note_id INTEGER PRIMARY KEY AUTOINCREMENT, + note text UNIQUE, + priority int + ) + ; + ', + NULL +) +ON CONFLICT (table_name) DO NOTHING +; + +INSERT INTO "__iql__.tables.columns" ( + table_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_notes', + 'note', + 'text', + 1, + 25, -- oid for text + 0, + 0 +) +ON CONFLICT (table_name, column_name) DO NOTHING +; + +INSERT INTO "__iql__.tables.columns" ( + table_name, + column_name, + column_type, + ordinal_position, + "oid", + column_width, + column_precision +) +VALUES ( + 'stackql_notes', + 'priority', + 'int', + 2, + 1700, -- oid for numeric + 0, + 0 +) +ON CONFLICT (table_name, column_name) DO NOTHING +; + +create table if not exists stackql_notes( + note_id INTEGER PRIMARY KEY AUTOINCREMENT, + note text UNIQUE, + priority int +) +; + +INSERT INTO stackql_notes(note, priority) +select 'v0.5.418 introduced table valued functions, for example json_each.' as note, 1000 as priority +union all +select 'stackql supports the postgres wire protocol.' as note, 10 as priority +on conflict (note) do nothing +; diff --git a/internal/stackql/sqlengine/sqlengine.go b/internal/stackql/sqlengine/sqlengine.go new file mode 100644 index 000000000..b77c77534 --- /dev/null +++ b/internal/stackql/sqlengine/sqlengine.go @@ -0,0 +1,51 @@ +package sqlengine + +import ( + "database/sql" + "fmt" + "path/filepath" + "runtime" + "strings" + + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" +) + +func getFilePathFromRepositoryRoot(relativePath string) (string, error) { + _, filename, _, _ := runtime.Caller(0) + curDir := filepath.Dir(filename) + rv, err := filepath.Abs(filepath.Join(curDir, "../../..", relativePath)) + return strings.ReplaceAll(rv, `\`, `\\`), err +} + +type SQLEngine interface { + GetDB() (*sql.DB, error) + GetTx() (*sql.Tx, error) + Exec(string, ...interface{}) (sql.Result, error) + Query(string, ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row + // ExecFileLocal(string) error + // ExecFile(string) error + ExecInTxn(queries []string) error + GetCurrentGenerationID() (int, error) + GetNextGenerationID() (int, error) + GetCurrentSessionID(int) (int, error) + GetNextSessionID(int) (int, error) + GetCurrentDiscoveryGenerationID(discoveryID string) (int, error) + GetNextDiscoveryGenerationID(discoveryID string) (int, error) + CacheStoreGet(string) ([]byte, error) + CacheStorePut(string, []byte, string, int) error + IsMemory() bool +} + +func NewSQLEngine(cfg Config, controlAttributes sqlcontrol.ControlAttributes) (SQLEngine, error) { + switch cfg.DBEngine { + case "sqlite3_embedded": + return newSQLiteEmbeddedEngine(cfg, controlAttributes) + case "postgres_tcp": + return newPostgresTCPEngine(cfg, controlAttributes) + case "snowflake": + return newSnowflakeTCPEngine(cfg, controlAttributes) + default: + return nil, fmt.Errorf(`SQL backend DB Engine of type '%s' is not permitted`, cfg.DBEngine) + } +} diff --git a/internal/stackql/sqlengine/sqlite_bool_decltype.go b/internal/stackql/sqlengine/sqlite_bool_decltype.go new file mode 100644 index 000000000..56a778246 --- /dev/null +++ b/internal/stackql/sqlengine/sqlite_bool_decltype.go @@ -0,0 +1,248 @@ +package sqlengine + +import ( + "context" + "database/sql/driver" + "fmt" + "reflect" + "strings" +) + +// boolDecltype is the declared column type that triggers boolean conversion; +// the retired cgo driver (mattn/go-sqlite3) compared the lowercased decltype +// against exactly "boolean". +const boolDecltype = "boolean" + +var ( + _ driver.Driver = (*boolDecltypeDriver)(nil) + _ moderncConn = (*boolDecltypeConn)(nil) + _ moderncStmt = (*boolDecltypeStmt)(nil) + _ moderncRows = (*boolDecltypeRows)(nil) +) + +// moderncConn is the optional-interface surface of the modernc.org/sqlite +// connection that database/sql consults; the wrapper delegates every one so +// pooling and cancellation behavior is unchanged. +type moderncConn interface { + driver.Conn + driver.ConnBeginTx + driver.ConnPrepareContext + driver.ExecerContext + driver.QueryerContext + driver.Pinger + driver.SessionResetter + driver.Validator +} + +// moderncStmt is the optional-interface surface of the modernc.org/sqlite +// prepared statement. +type moderncStmt interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +// moderncRows is the optional-interface surface of the modernc.org/sqlite +// rows iterator. +type moderncRows interface { + driver.Rows + driver.RowsColumnTypeDatabaseTypeName + driver.RowsColumnTypeLength + driver.RowsColumnTypeNullable + driver.RowsColumnTypePrecisionScale + driver.RowsColumnTypeScanType +} + +// boolDecltypeDriver restores mattn/go-sqlite3 parity for declared-boolean +// columns: INTEGER values in a column whose declared type is boolean +// (case-insensitive) surface as Go bool via the retired driver's val > 0 +// rule. modernc.org/sqlite performs decltype-based conversion for timestamps +// but not booleans, so such columns would otherwise surface as 0/1. +type boolDecltypeDriver struct { + inner driver.Driver +} + +func newBoolDecltypeDriver(inner driver.Driver) driver.Driver { + return &boolDecltypeDriver{inner: inner} +} + +func (d *boolDecltypeDriver) Open(name string) (driver.Conn, error) { + c, err := d.inner.Open(name) + if err != nil { + return nil, err + } + mc, ok := c.(moderncConn) + if !ok { + _ = c.Close() + return nil, fmt.Errorf("sqlite embedded: connection %T lacks the expected modernc.org/sqlite interfaces", c) + } + return &boolDecltypeConn{inner: mc}, nil +} + +type boolDecltypeConn struct { + inner moderncConn +} + +func (c *boolDecltypeConn) Prepare(query string) (driver.Stmt, error) { + return wrapBoolDecltypeStmt(c.inner.Prepare(query)) +} + +func (c *boolDecltypeConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + return wrapBoolDecltypeStmt(c.inner.PrepareContext(ctx, query)) +} + +func (c *boolDecltypeConn) Close() error { return c.inner.Close() } + +// Begin satisfies driver.Conn by delegating to BeginTx, so the deprecated +// inner method is never called; database/sql itself always prefers BeginTx. +func (c *boolDecltypeConn) Begin() (driver.Tx, error) { + return c.inner.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *boolDecltypeConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + return c.inner.BeginTx(ctx, opts) +} + +func (c *boolDecltypeConn) ExecContext( + ctx context.Context, + query string, + args []driver.NamedValue, +) (driver.Result, error) { + return c.inner.ExecContext(ctx, query, args) +} + +func (c *boolDecltypeConn) QueryContext( + ctx context.Context, + query string, + args []driver.NamedValue, +) (driver.Rows, error) { + return wrapBoolDecltypeRows(c.inner.QueryContext(ctx, query, args)) +} + +func (c *boolDecltypeConn) Ping(ctx context.Context) error { return c.inner.Ping(ctx) } + +func (c *boolDecltypeConn) ResetSession(ctx context.Context) error { return c.inner.ResetSession(ctx) } + +func (c *boolDecltypeConn) IsValid() bool { return c.inner.IsValid() } + +func wrapBoolDecltypeStmt(s driver.Stmt, err error) (driver.Stmt, error) { + if err != nil || s == nil { + return s, err + } + ms, ok := s.(moderncStmt) + if !ok { + _ = s.Close() + return nil, fmt.Errorf("sqlite embedded: statement %T lacks the expected modernc.org/sqlite interfaces", s) + } + return &boolDecltypeStmt{inner: ms}, nil +} + +type boolDecltypeStmt struct { + inner moderncStmt +} + +func (s *boolDecltypeStmt) Close() error { return s.inner.Close() } + +func (s *boolDecltypeStmt) NumInput() int { return s.inner.NumInput() } + +// Exec satisfies driver.Stmt by delegating to ExecContext, so the deprecated +// inner method is never called. +func (s *boolDecltypeStmt) Exec(args []driver.Value) (driver.Result, error) { + return s.inner.ExecContext(context.Background(), valuesToNamedValues(args)) +} + +// Query satisfies driver.Stmt by delegating to QueryContext, so the +// legacy inner method is never called. +func (s *boolDecltypeStmt) Query(args []driver.Value) (driver.Rows, error) { + return wrapBoolDecltypeRows(s.inner.QueryContext(context.Background(), valuesToNamedValues(args))) +} + +func (s *boolDecltypeStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return s.inner.ExecContext(ctx, args) +} + +func (s *boolDecltypeStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return wrapBoolDecltypeRows(s.inner.QueryContext(ctx, args)) +} + +func valuesToNamedValues(args []driver.Value) []driver.NamedValue { + named := make([]driver.NamedValue, len(args)) + for i, v := range args { + named[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return named +} + +// wrapBoolDecltypeRows wraps a result set that declares at least one boolean +// column; result sets without one pass through untouched. +func wrapBoolDecltypeRows(r driver.Rows, err error) (driver.Rows, error) { + if err != nil || r == nil { + return r, err + } + mr, ok := r.(moderncRows) + if !ok { + _ = r.Close() + return nil, fmt.Errorf("sqlite embedded: rows %T lack the expected modernc.org/sqlite interfaces", r) + } + var boolCols []bool + for i := range mr.Columns() { + if strings.EqualFold(mr.ColumnTypeDatabaseTypeName(i), boolDecltype) { + if boolCols == nil { + boolCols = make([]bool, len(mr.Columns())) + } + boolCols[i] = true + } + } + if boolCols == nil { + return mr, nil + } + return &boolDecltypeRows{inner: mr, boolCols: boolCols}, nil +} + +type boolDecltypeRows struct { + inner moderncRows + // boolCols marks the columns whose declared type is boolean. + boolCols []bool +} + +func (r *boolDecltypeRows) Columns() []string { return r.inner.Columns() } + +func (r *boolDecltypeRows) Close() error { return r.inner.Close() } + +// Next converts INTEGER values in declared-boolean columns to Go bool with +// the retired driver's val > 0 rule; NULL and every other storage class or +// column pass through unchanged. +func (r *boolDecltypeRows) Next(dest []driver.Value) error { + if err := r.inner.Next(dest); err != nil { + return err + } + for i, isBool := range r.boolCols { + if !isBool || i >= len(dest) { + continue + } + if v, ok := dest[i].(int64); ok { + dest[i] = v > 0 + } + } + return nil +} + +func (r *boolDecltypeRows) ColumnTypeDatabaseTypeName(index int) string { + return r.inner.ColumnTypeDatabaseTypeName(index) +} + +func (r *boolDecltypeRows) ColumnTypeLength(index int) (int64, bool) { + return r.inner.ColumnTypeLength(index) +} + +func (r *boolDecltypeRows) ColumnTypeNullable(index int) (bool, bool) { + return r.inner.ColumnTypeNullable(index) +} + +func (r *boolDecltypeRows) ColumnTypePrecisionScale(index int) (int64, int64, bool) { + return r.inner.ColumnTypePrecisionScale(index) +} + +func (r *boolDecltypeRows) ColumnTypeScanType(index int) reflect.Type { + return r.inner.ColumnTypeScanType(index) +} diff --git a/internal/stackql/sqlengine/sqlite_embedded.go b/internal/stackql/sqlengine/sqlite_embedded.go new file mode 100644 index 000000000..7698d57e0 --- /dev/null +++ b/internal/stackql/sqlengine/sqlite_embedded.go @@ -0,0 +1,614 @@ +package sqlengine + +import ( + "database/sql" + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + "github.com/stackql/stackql/internal/stackql/logging" + sqlcontrol "github.com/stackql/stackql/internal/stackql/sqlengine/control" + "github.com/stackql/stackql/internal/stackql/sqlengine/internal/sqlfuncs" + + sqlite "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +const ( + // sqliteDriverName publishes the modernc.org/sqlite driver with the + // StackQL extension functions registered. + sqliteDriverName = "stackql-sqlite" + sqliteDefaultDSN = "file::memory:?cache=shared" + // sqliteDefaultBusyTimeoutMillis preserves the retired cgo driver's + // per-connection default; modernc defaults to 0. + sqliteDefaultBusyTimeoutMillis = "5000" +) + +//nolint:gochecknoglobals // database/sql driver registration must occur once per process. +var ( + sqliteDriverRegisterOnce sync.Once + errSQLiteDriverRegistration error +) + +// registerSQLiteDriver idempotently publishes the sqlfuncs-equipped driver +// under sqliteDriverName, wrapped for mattn-parity boolean decltype +// conversion; a registration error surfaces on every attempt. +func registerSQLiteDriver() error { + sqliteDriverRegisterOnce.Do(func() { + drv := &sqlite.Driver{} + if err := sqlfuncs.Register(drv); err != nil { + errSQLiteDriverRegistration = err + return + } + sql.Register(sqliteDriverName, newBoolDecltypeDriver(drv)) + }) + return errSQLiteDriverRegistration +} + +var ( + _ SQLEngine = &sqLiteEmbeddedEngine{} +) + +type sqLiteEmbeddedEngine struct { + db *sql.DB + dsn string + controlAttributes sqlcontrol.ControlAttributes + ctrlMutex *sync.Mutex + sessionMutex *sync.Mutex + discoveryMutex *sync.Mutex +} + +func (se *sqLiteEmbeddedEngine) IsMemory() bool { + return strings.Contains(se.dsn, ":memory:") || strings.Contains(se.dsn, "mode=memory") +} + +func (se *sqLiteEmbeddedEngine) GetDB() (*sql.DB, error) { + return se.db, nil +} + +func (se *sqLiteEmbeddedEngine) GetTx() (*sql.Tx, error) { + return se.db.Begin() +} + +func newSQLiteEmbeddedEngine( + cfg Config, + controlAttributes sqlcontrol.ControlAttributes, +) (*sqLiteEmbeddedEngine, error) { + dsn, expectedPragmas, err := buildDSN(cfg.DSN) + eng := &sqLiteEmbeddedEngine{ + dsn: dsn, + controlAttributes: controlAttributes, + ctrlMutex: &sync.Mutex{}, + sessionMutex: &sync.Mutex{}, + discoveryMutex: &sync.Mutex{}, + } + if err != nil { + return eng, err + } + if err = registerSQLiteDriver(); err != nil { + return eng, err + } + db, err := sql.Open(sqliteDriverName, dsn) + db.SetConnMaxLifetime(-1) + eng.db = db + if err != nil { + return eng, err + } + if eng.IsMemory() && !strings.Contains(dsn, "cache=shared") { + // each pooled connection to a non-shared in-memory DSN would be a + // separate database + db.SetMaxOpenConns(1) + } + if err = eng.assertPragmas(expectedPragmas); err != nil { + return eng, err + } + if cfg.DbInitFilePath != "" { + err = eng.execFileSQLite(cfg.DbInitFilePath) + } + if err != nil { + return eng, err + } + logging.GetLogger().Infoln(fmt.Sprintf("opened db with file = '%s' and err = '%v'", dsn, err)) + return eng, err +} + +func (se *sqLiteEmbeddedEngine) execFileSQLite(fileName string) error { + fileContents, err := os.ReadFile(fileName) + if err != nil { + return err + } + _, err = se.db.Exec(string(fileContents)) + return err +} + +func (se *sqLiteEmbeddedEngine) execFileLocal(fileName string) error { + expF, err := getFilePathFromRepositoryRoot(fileName) + if err != nil { + return err + } + return se.execFileSQLite(expF) +} + +func (se *sqLiteEmbeddedEngine) ExecFileLocal(fileName string) error { + return se.execFileLocal(fileName) +} + +func (se *sqLiteEmbeddedEngine) ExecFile(fileName string) error { + return se.execFileSQLite(fileName) +} + +func (se sqLiteEmbeddedEngine) Exec(query string, varArgs ...interface{}) (sql.Result, error) { + // logging.GetLogger().Infoln(fmt.Sprintf("exec query = %s", query)) + res, err := se.db.Exec(query, varArgs...) + classifySQLiteError(err) + // logging.GetLogger().Infoln(fmt.Sprintf("res= %v, err = %v", res, err)) + return res, err +} + +func (se sqLiteEmbeddedEngine) ExecInTxn(queries []string) error { + txn, err := se.db.Begin() + if err != nil { + classifySQLiteError(err) + return err + } + for _, query := range queries { + _, err = txn.Exec(query) + if err != nil { + classifySQLiteError(err) + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + } + err = txn.Commit() + classifySQLiteError(err) + return err +} + +func (se sqLiteEmbeddedEngine) GetNextGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getNextGenerationID() +} + +func (se sqLiteEmbeddedEngine) GetCurrentGenerationID() (int, error) { + se.ctrlMutex.Lock() + defer se.ctrlMutex.Unlock() + return se.getCurrentGenerationID() +} + +func (se sqLiteEmbeddedEngine) GetNextDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getNextProviderGenerationID(discoveryName) +} + +func (se sqLiteEmbeddedEngine) GetCurrentDiscoveryGenerationID(discoveryName string) (int, error) { + se.discoveryMutex.Lock() + defer se.discoveryMutex.Unlock() + return se.getCurrentProviderGenerationID(discoveryName) +} + +func (se sqLiteEmbeddedEngine) GetNextSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getNextSessionID(generationID) +} + +func (se sqLiteEmbeddedEngine) GetCurrentSessionID(generationID int) (int, error) { + se.sessionMutex.Lock() + defer se.sessionMutex.Unlock() + return se.getCurrentSessionID(generationID) +} + +func (se sqLiteEmbeddedEngine) getCurrentGenerationID() (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_generation_id FROM "__iql__.control.generation" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.generation" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm WHERE lhs.collected_dttm IS null`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) QueryRow(query string, varArgs ...interface{}) *sql.Row { + res := se.db.QueryRow(query, varArgs...) + return res +} + +func (se sqLiteEmbeddedEngine) getNextGenerationID() (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.generation" (generation_description, created_dttm) VALUES ('', strftime('%s', 'now')) RETURNING iql_generation_id`) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) getCurrentProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_discovery_generation_id FROM "__iql__.control.discovery_generation" lhs INNER JOIN (SELECT discovery_name, max(created_dttm) AS max_dttm FROM "__iql__.control.discovery_generation" WHERE collected_dttm IS null GROUP BY discovery_name) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.discovery_name = rhs.discovery_name WHERE lhs.collected_dttm IS null AND lhs.discovery_name = ?`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) getNextProviderGenerationID(providerName string) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.discovery_generation" (discovery_name, created_dttm) VALUES (?, strftime('%s', 'now')) RETURNING iql_discovery_generation_id`, providerName) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) getCurrentSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll // long SQL query + res := se.db.QueryRow(`SELECT lhs.iql_session_id FROM "__iql__.control.session" lhs INNER JOIN (SELECT max(created_dttm) AS max_dttm FROM "__iql__.control.session" WHERE collected_dttm IS null) rhs ON lhs.created_dttm = rhs.max_dttm AND lhs.iql_genration_id = rhs.iql_generation_id WHERE lhs.iql_generation_id = ? AND lhs.collected_dttm IS null`, generationID) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) getNextSessionID(generationID int) (int, error) { + var retVal int + //nolint:lll,execinquery // long SQL query and `execinquery` is DEAD SET RUBBISH for INSERT... RETURNING + res := se.db.QueryRow(`INSERT INTO "__iql__.control.session" (iql_generation_id, created_dttm) VALUES (?, strftime('%s', 'now')) RETURNING iql_session_id`, generationID) + err := res.Scan(&retVal) + logging.GetLogger().Infoln( + fmt.Sprintf( + "getNextSessionID(): generation id = %d, session id = %d", + generationID, + retVal, + ), + ) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) CacheStoreGet(key string) ([]byte, error) { + var retVal []byte + res := se.db.QueryRow(`SELECT v FROM "__iql__.cache.key_val" WHERE k = ?`, key) + err := res.Scan(&retVal) + return retVal, err +} + +func (se sqLiteEmbeddedEngine) CacheStorePut(key string, val []byte, tablespace string, tablespaceID int) error { + txn, err := se.db.Begin() + if err != nil { + return err + } + _, err = txn.Exec(`DELETE FROM "__iql__.cache.key_val" WHERE k = ?`, key) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + _, err = txn.Exec( + `INSERT INTO "__iql__.cache.key_val" (k, v, tablespace, tablespace_id) VALUES(?, ?, ?, ?)`, + key, + val, + tablespace, + tablespaceID, + ) + if err != nil { + //nolint:errcheck // intentionally ignoring error TODO: publish variadic error(s) + txn.Rollback() + return err + } + err = txn.Commit() + return err +} + +func (se sqLiteEmbeddedEngine) Query(query string, varArgs ...interface{}) (*sql.Rows, error) { + return se.query(query, varArgs...) +} + +func (se sqLiteEmbeddedEngine) query(query string, varArgs ...interface{}) (*sql.Rows, error) { + logging.GetLogger().Debugln(fmt.Sprintf("sqlite embedded raw query = %s, varArgs = %v", query, varArgs)) + res, err := se.db.Query(query, varArgs...) + // logging.GetLogger().Infoln(fmt.Sprintf("res= %v, err = %v", res, err)) + return res, err +} + +// isBusy reports whether err carries primary result code SQLITE_BUSY. +func isBusy(err error) bool { + return hasSQLitePrimaryCode(err, sqlite3.SQLITE_BUSY) +} + +// isConstraintViolation reports whether err carries primary result code +// SQLITE_CONSTRAINT. +func isConstraintViolation(err error) bool { + return hasSQLitePrimaryCode(err, sqlite3.SQLITE_CONSTRAINT) +} + +// hasSQLitePrimaryCode compares the primary (low byte) result code; the +// driver enables extended result codes. +func hasSQLitePrimaryCode(err error, code int) bool { + var sqliteErr *sqlite.Error + if errors.As(err, &sqliteErr) { + return sqliteErr.Code()&0xff == code + } + return false +} + +// classifySQLiteError logs whether a write-path error is lock contention or +// a constraint failure. +func classifySQLiteError(err error) { + if err == nil { + return + } + switch { + case isBusy(err): + logging.GetLogger().Debugln(fmt.Sprintf("sqlite embedded: busy: %v", err)) + case isConstraintViolation(err): + logging.GetLogger().Debugln(fmt.Sprintf("sqlite embedded: constraint violation: %v", err)) + } +} + +// legacyParamTranslations maps the retired cgo driver's DSN shorthands to +// pragmas. Order matters: a later alias overrides its primary form, matching +// legacy precedence. modernc silently ignores unknown shorthands, so every +// one is translated explicitly. +// +//nolint:gochecknoglobals // Immutable driver configuration lookup. +var legacyParamTranslations = []struct { + param string + pragma string +}{ + {"_busy_timeout", "busy_timeout"}, + {"_timeout", "busy_timeout"}, + {"_journal_mode", "journal_mode"}, + {"_journal", "journal_mode"}, + {"_synchronous", "synchronous"}, + {"_sync", "synchronous"}, + {"_foreign_keys", "foreign_keys"}, + {"_fk", "foreign_keys"}, + {"_auto_vacuum", "auto_vacuum"}, + {"_vacuum", "auto_vacuum"}, + {"_cache_size", "cache_size"}, + {"_case_sensitive_like", "case_sensitive_like"}, + {"_cslike", "case_sensitive_like"}, + {"_defer_foreign_keys", "defer_foreign_keys"}, + {"_defer_fk", "defer_foreign_keys"}, + {"_ignore_check_constraints", "ignore_check_constraints"}, + {"_query_only", "query_only"}, + {"_recursive_triggers", "recursive_triggers"}, + {"_rt", "recursive_triggers"}, + {"_secure_delete", "secure_delete"}, + {"_writable_schema", "writable_schema"}, +} + +// moderncNativeParams pass through buildDSN verbatim. +// +//nolint:gochecknoglobals // Immutable driver configuration lookup. +var moderncNativeParams = map[string]struct{}{ + "_pragma": {}, + "_time_format": {}, + "_time_integer_format": {}, + "_timezone": {}, + "_txlock": {}, + "_inttotime": {}, + "_texttotime": {}, + "_defensive": {}, + "_error_rc": {}, +} + +// nonQueryablePragmas cannot be read back, so the startup assertion skips +// them. +// +//nolint:gochecknoglobals // Immutable driver configuration lookup. +var nonQueryablePragmas = map[string]struct{}{ + "case_sensitive_like": {}, +} + +var pragmaNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// buildDSN is the single place embedded-engine DSNs are constructed. It +// translates legacy `_param=value` shorthands to modernc `_pragma=name(value)` +// directives, passes through modernc-native and plain URI parameters, rejects +// unknown underscore parameters (modernc would silently ignore them), and +// injects the 5000ms legacy busy_timeout default when unset. An empty dsn +// selects the default shared in-memory database. The returned map records +// every pragma the DSN sets, for the startup assertion. +// +//nolint:gocognit,funlen // Preserve the migrated DSN translation rules during backend relocation. +func buildDSN(dsn string) (string, map[string]string, error) { + if dsn == "" { + dsn = sqliteDefaultDSN + } + base, rawQuery, _ := strings.Cut(dsn, "?") + query, err := url.ParseQuery(rawQuery) + if err != nil { + return "", nil, fmt.Errorf("invalid sqlite DSN query %q: %w", rawQuery, err) + } + expectedPragmas := map[string]string{} + translatedPragmas := map[string]string{} + outParams := url.Values{} + for key, vals := range query { + if len(vals) == 0 { + continue + } + if key == "_pragma" { + for _, directive := range vals { + name, value, valued := parsePragmaDirective(directive) + if valued { + expectedPragmas[name] = value + } + } + outParams["_pragma"] = append([]string{}, vals...) + continue + } + if _, isNative := moderncNativeParams[key]; isNative { + outParams[key] = vals + continue + } + if key == "_loc" { + // legacy time-location parameter; maps to _timezone, "auto" + // meaning the process-local zone + if query.Has("_timezone") { + return "", nil, fmt.Errorf("conflicting sqlite DSN parameters: _loc and _timezone") + } + loc := vals[len(vals)-1] + if strings.EqualFold(loc, "auto") { + loc = "Local" + } + outParams.Set("_timezone", loc) + continue + } + if isLegacyParam(key) { + continue // handled in declaration order below + } + if strings.HasPrefix(key, "_") { + return "", nil, fmt.Errorf("unsupported sqlite DSN parameter %q: no modernc.org/sqlite translation", key) + } + outParams[key] = vals + } + // declaration order makes an alias override its primary form + for _, tr := range legacyParamTranslations { + if vals := query[tr.param]; len(vals) > 0 { + translatedPragmas[tr.pragma] = vals[len(vals)-1] + } + } + for name, value := range translatedPragmas { + expectedPragmas[name] = value + } + if _, hasBusy := expectedPragmas["busy_timeout"]; !hasBusy { + translatedPragmas["busy_timeout"] = sqliteDefaultBusyTimeoutMillis + expectedPragmas["busy_timeout"] = sqliteDefaultBusyTimeoutMillis + } + translatedNames := make([]string, 0, len(translatedPragmas)) + for name := range translatedPragmas { + translatedNames = append(translatedNames, name) + } + sort.Strings(translatedNames) + directives := make([]string, 0, len(translatedNames)+len(outParams["_pragma"])) + for _, name := range translatedNames { + directives = append(directives, fmt.Sprintf("%s(%s)", name, translatedPragmas[name])) + } + directives = append(directives, outParams["_pragma"]...) + outParams["_pragma"] = directives + return base + "?" + outParams.Encode(), expectedPragmas, nil +} + +// parsePragmaDirective splits a `name(value)` or bare `name` directive; +// valued reports whether a value was supplied. +func parsePragmaDirective(directive string) (string, string, bool) { + name, rest, found := strings.Cut(directive, "(") + name = strings.TrimSpace(name) + if !found { + return name, "", false + } + value := strings.TrimSuffix(strings.TrimSpace(rest), ")") + return name, value, true +} + +func isLegacyParam(key string) bool { + for _, tr := range legacyParamTranslations { + if tr.param == key { + return true + } + } + return false +} + +// canonicalizePragmaValue reduces equivalent pragma value spellings to one +// form so requested and reported values compare. +func canonicalizePragmaValue(name, value string) string { + v := strings.ToLower(strings.TrimSpace(value)) + switch name { + case "foreign_keys", "defer_foreign_keys", "ignore_check_constraints", + "query_only", "recursive_triggers", "writable_schema", "case_sensitive_like": + switch v { + case "1", "true", "yes", "on": + return "1" + case "0", "false", "no", "off": + return "0" + } + case "synchronous": + switch v { + case "off": + return "0" + case "normal": + return "1" + case "full": + return "2" + case "extra": + return "3" + } + case "auto_vacuum": + switch v { + case "none": + return "0" + case "full": + return "1" + case "incremental": + return "2" + } + case "secure_delete": + switch v { + case "off", "false": + return "0" + case "on", "true": + return "1" + case "fast": + return "2" + } + } + return v +} + +// assertPragmas reads back every pragma the DSN set and fails fast on +// mismatch, so a translation that stops taking effect cannot fail silently. +func (se *sqLiteEmbeddedEngine) assertPragmas(expected map[string]string) error { + names := make([]string, 0, len(expected)) + for name := range expected { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, skip := nonQueryablePragmas[name]; skip { + continue + } + if !pragmaNamePattern.MatchString(name) { + return fmt.Errorf("pragma assertion: invalid pragma name %q", name) + } + var raw any + err := se.db.QueryRow("PRAGMA " + name).Scan(&raw) + if errors.Is(err, sql.ErrNoRows) { + // command-style pragmas report nothing + continue + } + if err != nil { + return fmt.Errorf("pragma assertion: cannot query pragma %q: %w", name, err) + } + actual := canonicalizePragmaValue(name, pragmaValueToString(raw)) + want := canonicalizePragmaValue(name, expected[name]) + if actual != want { + if name == "journal_mode" && se.IsMemory() && actual == "memory" { + // SQLite coerces in-memory journal mode to "memory" + continue + } + return fmt.Errorf("pragma assertion failed: %q is %q, requested %q", name, actual, want) + } + } + return nil +} + +func pragmaValueToString(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + case []byte: + return string(t) + case int64: + return strconv.FormatInt(t, 10) + case float64: + return strconv.FormatFloat(t, 'g', -1, 64) + default: + return fmt.Sprintf("%v", t) + } +} diff --git a/internal/stackql/sqlengine/sqlite_embedded_test.go b/internal/stackql/sqlengine/sqlite_embedded_test.go new file mode 100644 index 000000000..e655cb35e --- /dev/null +++ b/internal/stackql/sqlengine/sqlite_embedded_test.go @@ -0,0 +1,452 @@ +package sqlengine //nolint:testpackage // Exercises private driver and DSN helpers. + +import ( + "database/sql" + "errors" + "fmt" + "net/url" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func newTestEngine(t *testing.T, dsn string) *sqLiteEmbeddedEngine { + t.Helper() + eng, err := newSQLiteEmbeddedEngine(Config{DSN: dsn}, nil) + if err != nil { + t.Fatalf("cannot construct embedded engine for dsn %q: %v", dsn, err) + } + t.Cleanup(func() { + if eng.db != nil { + _ = eng.db.Close() + } + }) + return eng +} + +func TestBuildDSNDefault(t *testing.T) { + dsn, pragmas, err := buildDSN("") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.HasPrefix(dsn, "file::memory:?") { + t.Fatalf("default DSN base wrong: %q", dsn) + } + if !strings.Contains(dsn, "cache=shared") { + t.Fatalf("default DSN must preserve cache=shared: %q", dsn) + } + if pragmas["busy_timeout"] != "5000" { + t.Fatalf("default busy_timeout not injected: %v", pragmas) + } +} + +//nolint:gocognit // Preserve the migrated backend regression scenarios. +func TestBuildDSNTranslations(t *testing.T) { + cases := []struct { + name string + in string + wantPragmas map[string]string + wantParamSubstr []string + }{ + { + name: "legacy shorthands", + in: "file:test.db?_busy_timeout=1234&_journal_mode=WAL&_synchronous=NORMAL&_foreign_keys=on", + wantPragmas: map[string]string{ + "busy_timeout": "1234", + "journal_mode": "WAL", + "synchronous": "NORMAL", + "foreign_keys": "on", + }, + wantParamSubstr: []string{ + "busy_timeout(1234)", "journal_mode(WAL)", "synchronous(NORMAL)", "foreign_keys(on)", + }, + }, + { + name: "aliases win over primary forms", + in: "file:test.db?_busy_timeout=1&_timeout=9&_journal_mode=DELETE&_journal=WAL", + wantPragmas: map[string]string{ + "busy_timeout": "9", + "journal_mode": "WAL", + }, + wantParamSubstr: []string{"busy_timeout(9)", "journal_mode(WAL)"}, + }, + { + name: "native pragma directives pass through", + in: "file:test.db?_pragma=busy_timeout(2500)&_pragma=journal_mode(memory)", + wantPragmas: map[string]string{ + "busy_timeout": "2500", + "journal_mode": "memory", + }, + wantParamSubstr: []string{"busy_timeout(2500)", "journal_mode(memory)"}, + }, + { + name: "uri params pass through and busy_timeout injected", + in: "file::memory:?cache=shared&mode=memory", + wantPragmas: map[string]string{"busy_timeout": "5000"}, + wantParamSubstr: []string{ + "cache=shared", "mode=memory", "busy_timeout(5000)", + }, + }, + { + name: "extended legacy pragmas translate", + in: "file:test.db?_cache_size=-2000&_recursive_triggers=true&_secure_delete=FAST", + wantPragmas: map[string]string{"busy_timeout": "5000", "cache_size": "-2000", "recursive_triggers": "true", "secure_delete": "FAST"}, + wantParamSubstr: []string{ + "cache_size(-2000)", "recursive_triggers(true)", "secure_delete(FAST)", + }, + }, + { + name: "loc translates to timezone", + in: "file:test.db?_loc=auto", + wantPragmas: map[string]string{"busy_timeout": "5000"}, + wantParamSubstr: []string{"_timezone=Local"}, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + dsn, pragmas, err := buildDSN(tc.in) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for name, want := range tc.wantPragmas { + if got := pragmas[name]; got != want { + t.Fatalf("expected pragma %q=%q, got %q (map %v)", name, want, got, pragmas) + } + } + decoded, decodeErr := url.QueryUnescape(dsn) + if decodeErr != nil { + t.Fatalf("cannot decode generated DSN %q: %v", dsn, decodeErr) + } + for _, want := range tc.wantParamSubstr { + if !strings.Contains(decoded, want) { + t.Fatalf("generated DSN %q lacks %q", decoded, want) + } + } + }) + } +} + +func TestBuildDSNRejectsUnknownUnderscoreParam(t *testing.T) { + for _, dsn := range []string{ + "file:test.db?_mutex=full", + "file:test.db?_auth_user=admin", + "file:test.db?_no_such_param=1", + } { + if _, _, err := buildDSN(dsn); err == nil { + t.Fatalf("buildDSN(%q) must reject unknown underscore parameters", dsn) + } + } +} + +func TestSQLiteEmbeddedExtensionFunctions(t *testing.T) { + eng := newTestEngine(t, "") + assertQuery := func(query string, want any) { + t.Helper() + var got any + if err := eng.QueryRow(query).Scan(&got); err != nil { + t.Fatalf("%s: %v", query, err) + } + if fmt.Sprintf("%v", got) != fmt.Sprintf("%v", want) { + t.Fatalf("%s = %#v, want %#v", query, got, want) + } + } + assertQuery(`SELECT split_part('a,b,c', ',', 2)`, "b") + assertQuery(`SELECT regexp_like('abc123', '\d+')`, int64(1)) + assertQuery(`SELECT 'the year is 2021' REGEXP '[0-9]+'`, int64(1)) + assertQuery(`SELECT regexp_substr('abc123def', '\d+')`, "123") + assertQuery(`SELECT regexp_replace('abc123', '\d', 'X')`, "abcXXX") + assertQuery(`SELECT json_equal('{"a":1}', '{ "a" : 1.0 }')`, int64(1)) + assertQuery( + `SELECT aws_policy_equal('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":"*"}]}', '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}')`, + int64(1), + ) +} + +func TestSQLiteEmbeddedPragmaApplication(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "pragma.db") + eng := newTestEngine(t, + "file:"+dbPath+"?_busy_timeout=1234&_journal_mode=WAL&_synchronous=NORMAL&_foreign_keys=on") + assertPragma := func(name, want string) { + t.Helper() + var got any + if err := eng.QueryRow("PRAGMA " + name).Scan(&got); err != nil { + t.Fatalf("PRAGMA %s: %v", name, err) + } + if canon := canonicalizePragmaValue(name, pragmaValueToString(got)); canon != want { + t.Fatalf("PRAGMA %s = %q, want %q", name, canon, want) + } + } + assertPragma("busy_timeout", "1234") + assertPragma("journal_mode", "wal") + assertPragma("synchronous", "1") + assertPragma("foreign_keys", "1") +} + +func TestSQLiteEmbeddedPragmaAssertionFailure(t *testing.T) { + eng := newTestEngine(t, "") + err := eng.assertPragmas(map[string]string{"busy_timeout": "9999"}) + if err == nil { + t.Fatal("assertPragmas must fail fast on a pragma mismatch") + } + if !strings.Contains(err.Error(), "pragma assertion failed") { + t.Fatalf("unexpected assertion error: %v", err) + } +} + +func TestSQLiteEmbeddedPlainMemoryPoolCap(t *testing.T) { + eng := newTestEngine(t, ":memory:") + if !eng.IsMemory() { + t.Fatal("plain :memory: engine must report IsMemory") + } + if got := eng.db.Stats().MaxOpenConnections; got != 1 { + t.Fatalf("plain :memory: pool must be capped at 1 connection, got %d", got) + } + if _, err := eng.Exec(`CREATE TABLE cap_probe (v TEXT)`); err != nil { + t.Fatalf("create: %v", err) + } + var wg sync.WaitGroup + errCh := make(chan error, 16) + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + var c int + if err := eng.QueryRow(`SELECT count(*) FROM cap_probe`).Scan(&c); err != nil { + errCh <- err + } + }() + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Fatalf("concurrent query on capped memory pool failed: %v", err) + } +} + +func TestSQLiteEmbeddedSharedMemoryMultiConn(t *testing.T) { + eng := newTestEngine(t, "") + if !eng.IsMemory() { + t.Fatal("default engine must report IsMemory") + } + if got := eng.db.Stats().MaxOpenConnections; got != 0 { + t.Fatalf("shared-cache memory pool must stay uncapped, got %d", got) + } +} + +//nolint:gocognit // Preserve the migrated backend regression scenarios. +func TestSQLiteEmbeddedConcurrencyStress(t *testing.T) { + eng := newTestEngine(t, "") + if _, err := eng.Exec(`CREATE TABLE stress_t (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT)`); err != nil { + t.Fatalf("create: %v", err) + } + const ( + writers = 2 + readers = 8 + opsPerWorker = 200 + ) + var wg sync.WaitGroup + errCh := make(chan error, (writers+readers)*opsPerWorker) + for w := 0; w < writers; w++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for i := 0; i < opsPerWorker; i++ { + if _, err := eng.Exec(`INSERT INTO stress_t (v) VALUES (?)`, fmt.Sprintf("w%d-%d", n, i)); err != nil { + errCh <- fmt.Errorf("writer %d op %d: %w", n, i, err) + } + } + }(w) + } + for r := 0; r < readers; r++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for i := 0; i < opsPerWorker; i++ { + var c int + if err := eng.QueryRow(`SELECT count(*) FROM stress_t`).Scan(&c); err != nil { + errCh <- fmt.Errorf("reader %d op %d: %w", n, i, err) + } + } + }(r) + } + wg.Wait() + close(errCh) + for err := range errCh { + if isBusy(err) { + t.Fatalf("unhandled SQLITE_BUSY surfaced to a caller: %v", err) + } + t.Fatalf("concurrency stress error: %v", err) + } + var total int + if err := eng.QueryRow(`SELECT count(*) FROM stress_t`).Scan(&total); err != nil { + t.Fatalf("final count: %v", err) + } + if total != writers*opsPerWorker { + t.Fatalf("expected %d rows, got %d", writers*opsPerWorker, total) + } +} + +func TestSQLiteEmbeddedTimestampRoundTrip(t *testing.T) { + eng := newTestEngine(t, "") + ddl := `CREATE TABLE ts_probe ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_dttm INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + created_text TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + )` + if _, err := eng.Exec(ddl); err != nil { + t.Fatalf("create: %v", err) + } + before := time.Now().UTC().Add(-2 * time.Second).Unix() + if _, err := eng.Exec(`INSERT INTO ts_probe DEFAULT VALUES`); err != nil { + t.Fatalf("insert: %v", err) + } + after := time.Now().UTC().Add(2 * time.Second).Unix() + var epoch int64 + var text string + if err := eng.QueryRow(`SELECT created_dttm, created_text FROM ts_probe WHERE id = 1`).Scan(&epoch, &text); err != nil { + t.Fatalf("scan: %v", err) + } + if epoch < before || epoch > after { + t.Fatalf("epoch timestamp %d outside window [%d, %d]", epoch, before, after) + } + parsed, err := time.Parse("2006-01-02 15:04:05", text) + if err != nil { + t.Fatalf("CURRENT_TIMESTAMP text %q failed round trip: %v", text, err) + } + if u := parsed.UTC().Unix(); u < before || u > after { + t.Fatalf("text timestamp %q (%d) outside window [%d, %d]", text, u, before, after) + } + // Control-plane style INSERT ... RETURNING with strftime, as used by the + // generation/session bookkeeping queries. + var id int64 + if err := eng.QueryRow( + `INSERT INTO ts_probe (created_dttm) VALUES (strftime('%s', 'now')) RETURNING id`, + ).Scan(&id); err != nil { + t.Fatalf("insert returning: %v", err) + } + if id != 2 { + t.Fatalf("expected returned id 2, got %d", id) + } +} + +//nolint:gocognit // Preserve the migrated backend regression scenarios. +func TestSQLiteEmbeddedBooleanDecltypeRoundTrip(t *testing.T) { + eng := newTestEngine(t, "") + ddl := `CREATE TABLE bool_probe ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + flag BOOLEAN, + flag_lower boolean, + plain INTEGER + )` + if _, err := eng.Exec(ddl); err != nil { + t.Fatalf("create: %v", err) + } + // 2 and -1 pin the retired driver's val > 0 rule. + if _, err := eng.Exec( + `INSERT INTO bool_probe (flag, flag_lower, plain) VALUES (1, 1, 1), (0, 0, 0), (NULL, NULL, NULL), (2, 2, 2), (-1, -1, -1)`, + ); err != nil { + t.Fatalf("insert: %v", err) + } + expected := []struct { + flag any + plain any + }{ + {true, int64(1)}, + {false, int64(0)}, + {nil, nil}, + {true, int64(2)}, + {false, int64(-1)}, + } + rows, err := eng.Query(`SELECT flag, flag_lower, plain FROM bool_probe ORDER BY id`) + if err != nil { + t.Fatalf("query: %v", err) + } + defer func() { _ = rows.Close() }() + i := 0 + for rows.Next() { + var flag, flagLower, plain any + if err = rows.Scan(&flag, &flagLower, &plain); err != nil { + t.Fatalf("scan row %d: %v", i, err) + } + if i >= len(expected) { + t.Fatalf("unexpected extra row %d", i) + } + if flag != expected[i].flag { + t.Fatalf("row %d: BOOLEAN column = %#v, want %#v", i, flag, expected[i].flag) + } + if flagLower != expected[i].flag { + t.Fatalf("row %d: boolean (lowercase decltype) column = %#v, want %#v", i, flagLower, expected[i].flag) + } + if plain != expected[i].plain { + t.Fatalf("row %d: plain INTEGER column = %#v, want %#v (must stay untouched)", i, plain, expected[i].plain) + } + i++ + } + if err = rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + if i != len(expected) { + t.Fatalf("expected %d rows, got %d", len(expected), i) + } + // Typed sinks: bool and sql.NullBool round-trip, NULL preserved. + var b bool + if err = eng.QueryRow(`SELECT flag FROM bool_probe WHERE id = 1`).Scan(&b); err != nil || !b { + t.Fatalf("bool scan = (%v, %v), want (true, nil)", b, err) + } + var nb sql.NullBool + if err = eng.QueryRow(`SELECT flag FROM bool_probe WHERE id = 3`).Scan(&nb); err != nil { + t.Fatalf("NullBool scan: %v", err) + } + if nb.Valid { + t.Fatalf("NULL in a declared-boolean column must stay NULL, got %+v", nb) + } + // Prepared-statement path goes through the same conversion. + stmt, err := eng.db.Prepare(`SELECT flag FROM bool_probe WHERE id = ?`) + if err != nil { + t.Fatalf("prepare: %v", err) + } + defer func() { _ = stmt.Close() }() + var prepared any + if err = stmt.QueryRow(2).Scan(&prepared); err != nil { + t.Fatalf("prepared scan: %v", err) + } + if prepared != false { + t.Fatalf("prepared statement BOOLEAN column = %#v, want false", prepared) + } + // An expression over a boolean column has no decltype and must stay + // an INTEGER, exactly as under the retired driver. + var expr any + if err = eng.QueryRow(`SELECT flag + 0 FROM bool_probe WHERE id = 1`).Scan(&expr); err != nil { + t.Fatalf("expression scan: %v", err) + } + if expr != int64(1) { + t.Fatalf("expression column = %#v, want int64(1) (must stay untouched)", expr) + } +} + +func TestSQLiteErrorPredicates(t *testing.T) { + eng := newTestEngine(t, "") + if _, err := eng.Exec(`CREATE TABLE pred_t (id INTEGER PRIMARY KEY, v TEXT)`); err != nil { + t.Fatalf("create: %v", err) + } + if _, err := eng.Exec(`INSERT INTO pred_t (id, v) VALUES (1, 'a')`); err != nil { + t.Fatalf("insert: %v", err) + } + _, err := eng.Exec(`INSERT INTO pred_t (id, v) VALUES (1, 'dup')`) + if err == nil { + t.Fatal("duplicate insert must fail") + } + if !isConstraintViolation(err) { + t.Fatalf("expected constraint violation classification for %v", err) + } + if isBusy(err) { + t.Fatalf("constraint violation misclassified as busy: %v", err) + } + plain := errors.New("not a sqlite error") + if isBusy(plain) || isConstraintViolation(plain) { + t.Fatal("non-driver errors must not classify as sqlite errors") + } +} diff --git a/internal/stackql/sqlrewrite/sqlrewrite.go b/internal/stackql/sqlrewrite/sqlrewrite.go deleted file mode 100644 index 5d1353d15..000000000 --- a/internal/stackql/sqlrewrite/sqlrewrite.go +++ /dev/null @@ -1,309 +0,0 @@ -package sqlrewrite - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/internal/stackql/typing" -) - -type SQLRewriteInput interface { //nolint:revive //TODO: review - GetNamespaceCollection() tablenamespace.Collection - GetDRMConfig() drm.Config - GetColumnDescriptors() []typing.RelationalColumn - GetHoistedOnClauseTables() []sqlparser.SQLNode - GetBaseControlCounters() internaldto.TxnControlCounters - GetFromString() string - GetIndirectContexts() []drm.PreparedStatementCtx - GetPrepStmtOffset() int - GetSelectQualifier() string - GetSelectSuffix() string - GetRewrittenWhere() string - GetSecondaryCtrlCounters() []internaldto.TxnControlCounters - GetTables() taxonomy.TblMap - GetTableInsertionContainers() []tableinsertioncontainer.TableInsertionContainer - WithIndirectContexts(indirectContexts []drm.PreparedStatementCtx) SQLRewriteInput - WithPrepStmtOffset(offset int) SQLRewriteInput - GetParameters() map[string]interface{} - WithSelectQualifier(string) SQLRewriteInput - GetAliasToControlCounterMap() map[string][]internaldto.TxnControlCounters -} - -type standardSQLRewriteInput struct { - dc drm.Config - columnDescriptors []typing.RelationalColumn - baseControlCounters internaldto.TxnControlCounters - selectQualifier string - selectSuffix string - rewrittenWhere string - secondaryCtrlCounters []internaldto.TxnControlCounters - tables taxonomy.TblMap - fromString string - tableInsertionContainers []tableinsertioncontainer.TableInsertionContainer - namespaceCollection tablenamespace.Collection - indirectContexts []drm.PreparedStatementCtx - prepStmtOffset int - hoistedOnClauseTables []sqlparser.SQLNode - parameters map[string]interface{} -} - -func NewStandardSQLRewriteInput( - dc drm.Config, - columnDescriptors []typing.RelationalColumn, - baseControlCounters internaldto.TxnControlCounters, - selectSuffix string, - rewrittenWhere string, - secondaryCtrlCounters []internaldto.TxnControlCounters, - tables taxonomy.TblMap, - fromString string, - tableInsertionContainers []tableinsertioncontainer.TableInsertionContainer, - namespaceCollection tablenamespace.Collection, - hoistedOnClauseTables []sqlparser.SQLNode, - parameters map[string]interface{}, -) SQLRewriteInput { - return &standardSQLRewriteInput{ - dc: dc, - columnDescriptors: columnDescriptors, - baseControlCounters: baseControlCounters, - selectSuffix: selectSuffix, - rewrittenWhere: rewrittenWhere, - secondaryCtrlCounters: secondaryCtrlCounters, - tables: tables, - fromString: fromString, - tableInsertionContainers: tableInsertionContainers, - namespaceCollection: namespaceCollection, - hoistedOnClauseTables: hoistedOnClauseTables, - parameters: parameters, - } -} - -func (ri *standardSQLRewriteInput) GetHoistedOnClauseTables() []sqlparser.SQLNode { - return ri.hoistedOnClauseTables -} - -func (ri *standardSQLRewriteInput) WithSelectQualifier(selectQualifier string) SQLRewriteInput { - ri.selectQualifier = selectQualifier - return ri -} - -func (ri *standardSQLRewriteInput) GetSelectQualifier() string { - return ri.selectQualifier -} - -func (ri *standardSQLRewriteInput) GetPrepStmtOffset() int { - return ri.prepStmtOffset -} - -func (ri *standardSQLRewriteInput) GetParameters() map[string]interface{} { - return ri.parameters -} - -func (ri *standardSQLRewriteInput) WithPrepStmtOffset(offset int) SQLRewriteInput { - ri.prepStmtOffset = offset - return ri -} - -func (ri *standardSQLRewriteInput) GetDRMConfig() drm.Config { - return ri.dc -} - -func (ri *standardSQLRewriteInput) WithIndirectContexts(indirectContexts []drm.PreparedStatementCtx) SQLRewriteInput { - ri.indirectContexts = indirectContexts - return ri -} - -func (ri *standardSQLRewriteInput) GetIndirectContexts() []drm.PreparedStatementCtx { - return ri.indirectContexts -} - -func (ri *standardSQLRewriteInput) GetNamespaceCollection() tablenamespace.Collection { - return ri.namespaceCollection -} - -func (ri *standardSQLRewriteInput) GetColumnDescriptors() []typing.RelationalColumn { - return ri.columnDescriptors -} - -func (ri *standardSQLRewriteInput) GetTableInsertionContainers() []tableinsertioncontainer.TableInsertionContainer { - return ri.tableInsertionContainers -} - -func (ri *standardSQLRewriteInput) GetBaseControlCounters() internaldto.TxnControlCounters { - return ri.baseControlCounters -} - -func (ri *standardSQLRewriteInput) GetSelectSuffix() string { - return ri.selectSuffix -} - -func (ri *standardSQLRewriteInput) GetFromString() string { - return ri.fromString -} - -func (ri *standardSQLRewriteInput) GetRewrittenWhere() string { - return ri.rewrittenWhere -} - -func (ri *standardSQLRewriteInput) GetSecondaryCtrlCounters() []internaldto.TxnControlCounters { - return ri.secondaryCtrlCounters -} - -func (ri *standardSQLRewriteInput) GetTables() taxonomy.TblMap { - return ri.tables -} - -func (ri *standardSQLRewriteInput) GetAliasToControlCounterMap() map[string][]internaldto.TxnControlCounters { - aliasToControlCounterMap := make(map[string][]internaldto.TxnControlCounters) - for _, tbl := range ri.tableInsertionContainers { - tableName, controlCounter := tbl.GetTableTxnCounters() - _, ok := aliasToControlCounterMap[tableName] - if ok { - aliasToControlCounterMap[tableName] = append(aliasToControlCounterMap[tableName], controlCounter) - continue - } - aliasToControlCounterMap[tableName] = []internaldto.TxnControlCounters{controlCounter} - } - return aliasToControlCounterMap -} - -//nolint:funlen,gocognit //TODO: review -func GenerateRewrittenSelectDML(input SQLRewriteInput) (drm.PreparedStatementCtx, error) { - dc := input.GetDRMConfig() - cols := input.GetColumnDescriptors() - var txnCtrlCtrs internaldto.TxnControlCounters - var secondaryCtrlCounters []internaldto.TxnControlCounters - selectQualifier := input.GetSelectQualifier() - selectSuffix := input.GetSelectSuffix() - rewrittenWhere := input.GetRewrittenWhere() - var columns []typing.ColumnMetadata - var relationalColumns []typing.RelationalColumn - var tableAliases []string - for _, col := range cols { - relationalColumn := col - columns = append( - columns, - typing.NewRelayedColDescriptor( - relationalColumn, relationalColumn.GetType())) - // TODO: Need a way to handle postgres differences. This is a fragile point - relationalColumns = append(relationalColumns, relationalColumn) - } - genIDColName := dc.GetControlAttributes().GetControlGenIDColumnName() - sessionIDColName := dc.GetControlAttributes().GetControlSsnIDColumnName() - txnIDColName := dc.GetControlAttributes().GetControlTxnIDColumnName() - insIDColName := dc.GetControlAttributes().GetControlInsIDColumnName() - insEncodedColName := dc.GetControlAttributes().GetControlInsertEncodedIDColumnName() - inputContainers := input.GetTableInsertionContainers() - hoistedOnClauseTables := input.GetHoistedOnClauseTables() - hoistedTableAliases := make([]string, len(input.GetHoistedOnClauseTables())) - tblMap := input.GetTables() - if len(inputContainers) == 0 { - txnCtrlCtrs = input.GetBaseControlCounters() - secondaryCtrlCounters = input.GetSecondaryCtrlCounters() - } - i := 0 - // TODO: Only add control stuff to where clause if not - // already in - // an ON clause - // First pass; deal with ON clause hoisted tables - for _, tb := range inputContainers { - v := tb.GetTableMetadata() - isOnClauseHoistable := v.IsOnClauseHoistable() - if !isOnClauseHoistable { - continue - } - var aliasFound bool - var foundIdx int - for idx, node := range hoistedOnClauseTables { - t := tblMap[node] - if v.GetAlias() == t.GetAlias() { - hoistedTableAliases[idx] = t.GetAlias() - aliasFound = true - foundIdx = idx - break - } - } - if !aliasFound { - return nil, fmt.Errorf("could not find alias for hoisted table") - } - // This is required because of TOPO SORT - if foundIdx == 0 && txnCtrlCtrs == nil { - _, txnCtrlCtrs = tb.GetTableTxnCounters() - } - if foundIdx > 0 { - _, secondaryCtr := tb.GetTableTxnCounters() - secondaryCtrlCounters = append(secondaryCtrlCounters, secondaryCtr) - } - i++ - } - aliasCache := make(map[string]struct{}) - // Second pass; deal with non-hoisted tables - for _, tb := range inputContainers { - v := tb.GetTableMetadata() - isOnClauseHoistable := v.IsOnClauseHoistable() - if isOnClauseHoistable { - continue - } - // This is required because of TOPO SORT - // TODO: support in clause dataflow stuff - if txnCtrlCtrs == nil { - _, txnCtrlCtrs = tb.GetTableTxnCounters() - } - // TODO: fix this hack - // Alias is a marker for "inside insertion group" - alias := v.GetAlias() - _, aliasPresent := aliasCache[alias] - if i > 0 { - _, secondaryCtr := tb.GetTableTxnCounters() - secondaryCtrlCounters = append(secondaryCtrlCounters, secondaryCtr) - } - i++ - if aliasPresent { - continue - } - aliasCache[alias] = struct{}{} - tableAliases = append(tableAliases, alias) - } - - // TODO add in some handle for ON clause predicates - query, err := dc.GetSQLSystem().ComposeSelectQuery( - relationalColumns, tableAliases, hoistedTableAliases, input.GetFromString(), - rewrittenWhere, selectQualifier, selectSuffix, input.GetPrepStmtOffset(), - input.GetAliasToControlCounterMap(), - ) - if err != nil { - return nil, err - } - countedTables := 0 - for _, tbl := range input.GetTables() { - isMaterializedView := tbl.IsMaterializedView() - isPhysicalTable := tbl.IsPhysicalTable() - if tbl != nil && !isMaterializedView && !isPhysicalTable { - countedTables++ - } - } - // reorderedSecondaryCtrlCounters := reorderSliceAsRing(secondaryCtrlCounters, countedTables) - rv := drm.NewPreparedStatementCtx( - query, - "", - genIDColName, - sessionIDColName, - nil, - txnIDColName, - insIDColName, - insEncodedColName, - columns, - countedTables, - txnCtrlCtrs, - secondaryCtrlCounters, - input.GetDRMConfig().GetNamespaceCollection(), - dc.GetSQLSystem(), - input.GetParameters(), - ).WithAliasToTccMap(input.GetAliasToControlCounterMap()).WithAliasOrdering(tableAliases) - rv.SetIndirectContexts(input.GetIndirectContexts()) - return rv, nil -} diff --git a/internal/stackql/sqlrewrite/sqlrewrite_test.go b/internal/stackql/sqlrewrite/sqlrewrite_test.go deleted file mode 100644 index d8714e064..000000000 --- a/internal/stackql/sqlrewrite/sqlrewrite_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package sqlrewrite //nolint:testpackage - -import ( - "errors" - "reflect" - "testing" - - "github.com/stackql/any-sdk/pkg/db/sqlcontrol" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/sql_system" - "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/tablenamespace" - "github.com/stackql/stackql/internal/stackql/taxonomy" - "github.com/stackql/stackql/internal/stackql/typing" -) - -type fakeControlAttributes struct{ sqlcontrol.ControlAttributes } - -func (fakeControlAttributes) GetControlGenIDColumnName() string { return "gen_id" } -func (fakeControlAttributes) GetControlSsnIDColumnName() string { return "ssn_id" } -func (fakeControlAttributes) GetControlTxnIDColumnName() string { return "txn_id" } -func (fakeControlAttributes) GetControlInsIDColumnName() string { return "ins_id" } -func (fakeControlAttributes) GetControlInsertEncodedIDColumnName() string { return "ins_enc" } - -type fakeSQLSystem struct { - sql_system.SQLSystem - query, err string - aliases, hoisted, rest []string - offset int -} - -func (s *fakeSQLSystem) ComposeSelectQuery( - _ []typing.RelationalColumn, aliases, hoisted []string, from, where, qualifier, suffix string, - offset int, _ map[string][]internaldto.TxnControlCounters, -) (string, error) { - s.aliases, s.hoisted, s.rest, s.offset = aliases, hoisted, []string{from, where, qualifier, suffix}, offset - if s.err != "" { - return "", errors.New(s.err) - } - return s.query, nil -} - -type fakeDRMConfig struct { - drm.Config - sqlSystem *fakeSQLSystem -} - -func (c fakeDRMConfig) GetControlAttributes() sqlcontrol.ControlAttributes { - return fakeControlAttributes{} -} -func (c fakeDRMConfig) GetSQLSystem() sql_system.SQLSystem { return c.sqlSystem } -func (c fakeDRMConfig) GetNamespaceCollection() tablenamespace.Collection { return nil } - -type fakeTable struct { - tablemetadata.ExtendedTableMetadata - alias string - hoistable, physical, materialized bool -} - -func (m *fakeTable) GetAlias() string { return m.alias } -func (m *fakeTable) IsOnClauseHoistable() bool { return m.hoistable } -func (m *fakeTable) IsPhysicalTable() bool { return m.physical } -func (m *fakeTable) IsMaterializedView() bool { return m.materialized } - -type fakeContainer struct { - tableinsertioncontainer.TableInsertionContainer - tm *fakeTable - counters internaldto.TxnControlCounters -} - -func (c *fakeContainer) GetTableMetadata() tablemetadata.ExtendedTableMetadata { return c.tm } -func (c *fakeContainer) GetTableTxnCounters() (string, internaldto.TxnControlCounters) { - return c.tm.alias, c.counters -} - -func counters(n int) internaldto.TxnControlCounters { - return internaldto.NewTxnControlCountersFromVals(n, n, n, n) -} - -func newInput( - s *fakeSQLSystem, base internaldto.TxnControlCounters, secondary []internaldto.TxnControlCounters, - tables taxonomy.TblMap, containers []tableinsertioncontainer.TableInsertionContainer, hoisted []sqlparser.SQLNode, -) SQLRewriteInput { - return NewStandardSQLRewriteInput( - fakeDRMConfig{sqlSystem: s}, []typing.RelationalColumn{typing.NewRelationalColumn("name", "text")}, - base, "ORDER BY name", "name = 'x'", secondary, tables, "FROM t", containers, nil, hoisted, - map[string]interface{}{"p": 1}, - ) -} - -type want struct { - err string - query string - counters []internaldto.TxnControlCounters - repeats int - indirect int - aliases []string - hoisted []string - rest []string - offset int - aliasMap map[string][]internaldto.TxnControlCounters - orderedTccs []internaldto.TxnControlCounters -} - -func TestGenerateRewrittenSelectDML(t *testing.T) { - tests := []struct { - name string - input func(s *fakeSQLSystem) SQLRewriteInput - sys *fakeSQLSystem - want want - }{ - { - name: "without containers uses base counters", - sys: &fakeSQLSystem{query: "SELECT name FROM t"}, - input: func(s *fakeSQLSystem) SQLRewriteInput { - tables := taxonomy.TblMap{ - &sqlparser.AliasedTableExpr{}: &fakeTable{alias: "t"}, - &sqlparser.AliasedTableExpr{}: &fakeTable{alias: "p", physical: true}, - &sqlparser.AliasedTableExpr{}: &fakeTable{alias: "m", materialized: true}, - } - indirect := []drm.PreparedStatementCtx{drm.NewQueryOnlyPreparedStatementCtx("SELECT 1", nil)} - return newInput(s, counters(1), []internaldto.TxnControlCounters{counters(2)}, tables, nil, nil). - WithSelectQualifier("DISTINCT").WithPrepStmtOffset(2).WithIndirectContexts(indirect) - }, - want: want{ - query: "SELECT name FROM t", - counters: []internaldto.TxnControlCounters{counters(1), counters(2)}, - // Only table t is counted. Physical tables (p) and materialized views (m) are not. - repeats: 1, - indirect: 1, - aliases: []string{}, - hoisted: []string{}, - rest: []string{"FROM t", "name = 'x'", "DISTINCT", "ORDER BY name"}, - offset: 2, - }, - }, - { - name: "with containers and a hoisted table", - sys: &fakeSQLSystem{query: "SELECT 1"}, - input: func(s *fakeSQLSystem) SQLRewriteInput { - hoistedNode := &sqlparser.AliasedTableExpr{} - hoistedTable := &fakeTable{alias: "h", hoistable: true} - containers := []tableinsertioncontainer.TableInsertionContainer{ - &fakeContainer{tm: &fakeTable{alias: "a"}, counters: counters(2)}, - &fakeContainer{tm: hoistedTable, counters: counters(1)}, - &fakeContainer{tm: &fakeTable{alias: "a"}, counters: counters(3)}, - } - // When there are containers, the base counters (9) are not used. - return newInput(s, counters(9), nil, taxonomy.TblMap{hoistedNode: hoistedTable}, containers, - []sqlparser.SQLNode{hoistedNode}) - }, - want: want{ - query: "SELECT 1", - // The hoisted table (h) comes first, then the other tables in order. - counters: []internaldto.TxnControlCounters{counters(1), counters(2), counters(3)}, - repeats: 1, - hoisted: []string{"h"}, - // Alias a appears twice but is sent to ComposeSelectQuery once. - aliases: []string{"a"}, - rest: []string{"FROM t", "name = 'x'", "", "ORDER BY name"}, - // Counters for the same table are kept together in one list. - aliasMap: map[string][]internaldto.TxnControlCounters{ - "a": {counters(2), counters(3)}, - "h": {counters(1)}, - }, - orderedTccs: []internaldto.TxnControlCounters{counters(2), counters(3)}, - }, - }, - { - name: "hoisted table without a matching alias", - sys: &fakeSQLSystem{}, - input: func(s *fakeSQLSystem) SQLRewriteInput { - hoistedNode := &sqlparser.AliasedTableExpr{} - tables := taxonomy.TblMap{hoistedNode: &fakeTable{alias: "other", hoistable: true}} - containers := []tableinsertioncontainer.TableInsertionContainer{ - &fakeContainer{tm: &fakeTable{alias: "h", hoistable: true}}, - } - return newInput(s, nil, nil, tables, containers, []sqlparser.SQLNode{hoistedNode}) - }, - want: want{err: "could not find alias for hoisted table"}, - }, - { - name: "compose error is returned", - sys: &fakeSQLSystem{err: "compose failed"}, - input: func(s *fakeSQLSystem) SQLRewriteInput { - return newInput(s, nil, nil, nil, nil, nil) - }, - want: want{err: "compose failed"}, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - input := tc.input(tc.sys) - ctx, err := GenerateRewrittenSelectDML(input) - if tc.want.err != "" { - if err == nil || err.Error() != tc.want.err { - t.Fatalf("expected error %q, got %v", tc.want.err, err) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - checkContext(t, ctx, tc.want) - checkComposeArgs(t, tc.sys, tc.want) - if tc.want.aliasMap != nil && !reflect.DeepEqual(input.GetAliasToControlCounterMap(), tc.want.aliasMap) { - t.Errorf("unexpected alias to counter map %v", input.GetAliasToControlCounterMap()) - } - }) - } -} - -func checkContext(t *testing.T, ctx drm.PreparedStatementCtx, w want) { - t.Helper() - if ctx.GetQuery() != w.query { - t.Errorf("unexpected query %q", ctx.GetQuery()) - } - if !reflect.DeepEqual(ctx.GetAllCtrlCtrs(), w.counters) { - t.Errorf("unexpected control counters %v", ctx.GetAllCtrlCtrs()) - } - if ctx.GetCtrlColumnRepeats() != w.repeats { - t.Errorf("expected %d control column repeats, got %d", w.repeats, ctx.GetCtrlColumnRepeats()) - } - if len(ctx.GetNonControlColumns()) != 1 || len(ctx.GetIndirectContexts()) != w.indirect { - t.Error("expected columns and indirect contexts to be carried over") - } - if w.orderedTccs != nil && !reflect.DeepEqual(ctx.GetOrderedTccs(), w.orderedTccs) { - t.Errorf("unexpected ordered control counters %v", ctx.GetOrderedTccs()) - } -} - -func checkComposeArgs(t *testing.T, s *fakeSQLSystem, w want) { - t.Helper() - if !sameStrings(s.aliases, w.aliases) || !sameStrings(s.hoisted, w.hoisted) { - t.Errorf("unexpected aliases %v and hoisted aliases %v", s.aliases, s.hoisted) - } - if !reflect.DeepEqual(s.rest, w.rest) || s.offset != w.offset { - t.Errorf("unexpected ComposeSelectQuery arguments %v, offset %d", s.rest, s.offset) - } -} - -func sameStrings(a, b []string) bool { - if len(a) == 0 && len(b) == 0 { - return true - } - return reflect.DeepEqual(a, b) -} diff --git a/internal/stackql/sqlstream/sql_map_stream.go b/internal/stackql/sqlstream/sql_map_stream.go index be417091d..df745f3cc 100644 --- a/internal/stackql/sqlstream/sql_map_stream.go +++ b/internal/stackql/sqlstream/sql_map_stream.go @@ -4,10 +4,10 @@ import ( "fmt" "io" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/drm" + "github.com/stackql/stackql/internal/stackql/logging" + "github.com/stackql/stackql/internal/stackql/sqlengine" + "github.com/stackql/stackql/internal/stackql/streaming" "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" ) diff --git a/internal/stackql/sqlstream/static_map_stream.go b/internal/stackql/sqlstream/static_map_stream.go index 9cb04e77a..8c3f19b3a 100644 --- a/internal/stackql/sqlstream/static_map_stream.go +++ b/internal/stackql/sqlstream/static_map_stream.go @@ -3,7 +3,7 @@ package sqlstream import ( "io" - "github.com/stackql/any-sdk/pkg/streaming" + "github.com/stackql/stackql/internal/stackql/streaming" ) type StaticMapStream struct { diff --git a/internal/stackql/streaming/map_stream.go b/internal/stackql/streaming/map_stream.go new file mode 100644 index 000000000..2289b55cc --- /dev/null +++ b/internal/stackql/streaming/map_stream.go @@ -0,0 +1,48 @@ +package streaming + +import ( + "io" +) + +type StackQLReader interface{} + +type StackQLWriter interface{} + +type StackQLReadWriter interface { + StackQLReader + StackQLWriter +} + +type MapReader interface { + StackQLReader + Read() ([]map[string]interface{}, error) +} + +type MapWriter interface { + StackQLWriter + Write([]map[string]interface{}) error +} + +type standardMapStream struct { + store []map[string]interface{} +} + +type MapStream interface { + MapReader + MapWriter +} + +func NewstandardMapStream() MapStream { + return &standardMapStream{} +} + +func (ss *standardMapStream) Write(input []map[string]interface{}) error { + ss.store = append(ss.store, input...) + return nil +} + +func (ss *standardMapStream) Read() ([]map[string]interface{}, error) { + rv := ss.store + ss.store = nil + return rv, io.EOF +} diff --git a/internal/stackql/streaming/nop_map_stream.go b/internal/stackql/streaming/nop_map_stream.go new file mode 100644 index 000000000..18e3b8a9c --- /dev/null +++ b/internal/stackql/streaming/nop_map_stream.go @@ -0,0 +1,5 @@ +package streaming + +func NewNopMapStream() MapStream { + return &standardMapStream{} +} diff --git a/internal/stackql/suffix/suffix.go b/internal/stackql/suffix/suffix.go deleted file mode 100644 index 6fb0078a7..000000000 --- a/internal/stackql/suffix/suffix.go +++ /dev/null @@ -1,61 +0,0 @@ -package suffix - -import ( - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/go-suffix-map/pkg/suffixmap" -) - -var ( - _ ParameterSuffixMap = &standardParameterSuffixMap{} -) - -type ParameterSuffixMap interface { - Delete(k string) bool - Get(k string) (formulation.Addressable, bool) - GetAll() map[string]formulation.Addressable - Put(k string, v formulation.Addressable) - Size() int -} - -type standardParameterSuffixMap struct { - sm suffixmap.SuffixMap -} - -func NewParameterSuffixMap() ParameterSuffixMap { - return &standardParameterSuffixMap{ - sm: suffixmap.NewSuffixMap(nil), - } -} - -func (psm *standardParameterSuffixMap) Get(k string) (formulation.Addressable, bool) { - rv, ok := psm.sm.Get(k) - if !ok { - return nil, false - } - crv, ok := rv.(formulation.Addressable) - return crv, ok -} - -func (psm *standardParameterSuffixMap) GetAll() map[string]formulation.Addressable { - m := psm.sm.GetAll() - rv := make(map[string]formulation.Addressable) - for k, v := range m { - p, ok := v.(formulation.Addressable) - if ok { - rv[k] = p - } - } - return rv -} - -func (psm *standardParameterSuffixMap) Put(k string, v formulation.Addressable) { - psm.sm.Put(k, v) -} - -func (psm *standardParameterSuffixMap) Delete(k string) bool { - return psm.sm.Delete(k) -} - -func (psm *standardParameterSuffixMap) Size() int { - return psm.sm.Size() -} diff --git a/internal/stackql/suffix/suffix_test.go b/internal/stackql/suffix/suffix_test.go deleted file mode 100644 index 5cf14274b..000000000 --- a/internal/stackql/suffix/suffix_test.go +++ /dev/null @@ -1,231 +0,0 @@ -package suffix_test - -import ( - "testing" - - "github.com/stackql/stackql/internal/stackql/suffix" -) - -// mockAddressable implements formulation.Addressable for testing purposes. -type mockAddressable struct { - name string - typ string - conditionResult bool -} - -func (m *mockAddressable) ConditionIsValid(lhs string, rhs interface{}) bool { - return m.conditionResult -} - -func (m *mockAddressable) GetName() string { - return m.name -} - -func (m *mockAddressable) GetType() string { - return m.typ -} - -func newMock(name, typ string) *mockAddressable { - return &mockAddressable{name: name, typ: typ, conditionResult: true} -} - -func TestNewParameterSuffixMap(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - if psm == nil { - t.Fatal("NewParameterSuffixMap returned nil") - } -} - -func TestParameterSuffixMapSize(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - if got := psm.Size(); got != 0 { - t.Errorf("expected size 0, got %d", got) - } -} - -func TestParameterSuffixMapPut(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - if got := psm.Size(); got != 1 { - t.Errorf("expected size 1, got %d", got) - } -} - -func TestParameterSuffixMapPutMultiple(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - psm.Put("key2", newMock("name2", "type2")) - psm.Put("key3", newMock("name3", "type3")) - if got := psm.Size(); got != 3 { - t.Errorf("expected size 3, got %d", got) - } -} - -func TestParameterSuffixMapGet(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - - val, ok := psm.Get("key1") - if !ok { - t.Error("expected to find key1") - } - addr, ok := val.(*mockAddressable) - if !ok { - t.Fatalf("expected *mockAddressable, got %T", val) - } - if addr.name != "name1" { - t.Errorf("expected name1, got %s", addr.name) - } -} - -func TestParameterSuffixMapGetNonExistent(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - - _, ok := psm.Get("nonexistent") - if ok { - t.Error("expected not to find nonexistent key") - } -} - -func TestParameterSuffixMapGetAll(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - psm.Put("key2", newMock("name2", "type2")) - - all := psm.GetAll() - if len(all) != 2 { - t.Errorf("expected 2 items, got %d", len(all)) - } - if _, ok := all["key1"]; !ok { - t.Error("expected key1 in GetAll result") - } - if _, ok := all["key2"]; !ok { - t.Error("expected key2 in GetAll result") - } -} - -func TestParameterSuffixMapGetAllEmpty(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - all := psm.GetAll() - if len(all) != 0 { - t.Errorf("expected empty map, got %d items", len(all)) - } -} - -func TestParameterSuffixMapDelete(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - - if got := psm.Size(); got != 1 { - t.Fatalf("expected size 1, got %d", got) - } - - deleted := psm.Delete("key1") - if !deleted { - t.Error("expected Delete to return true") - } - - if got := psm.Size(); got != 0 { - t.Errorf("expected size 0 after delete, got %d", got) - } - - _, ok := psm.Get("key1") - if ok { - t.Error("expected key1 to be deleted") - } -} - -func TestParameterSuffixMapDeleteNonExistent(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - deleted := psm.Delete("nonexistent") - if deleted { - t.Error("expected Delete to return false for nonexistent key") - } -} - -func TestParameterSuffixMapDeleteUpdatesGetAll(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - psm.Put("key2", newMock("name2", "type2")) - - psm.Delete("key1") - all := psm.GetAll() - if len(all) != 1 { - t.Errorf("expected 1 item after delete, got %d", len(all)) - } - if _, ok := all["key1"]; ok { - t.Error("expected key1 to be absent after delete") - } - if _, ok := all["key2"]; !ok { - t.Error("expected key2 to remain after delete") - } -} - -func TestParameterSuffixMapPutOverwrite(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "type1")) - psm.Put("key1", newMock("name2", "type2")) - - all := psm.GetAll() - if len(all) != 1 { - t.Errorf("expected 1 item, got %d", len(all)) - } - - val, _ := psm.Get("key1") - addr := val.(*mockAddressable) - if addr.name != "name2" { - t.Errorf("expected name2, got %s", addr.name) - } -} - -func TestParameterSuffixMapConditionIsValid(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - // Test that we can store addressables with different ConditionIsValid results - trueAddr := &mockAddressable{name: "name1", typ: "type1", conditionResult: true} - falseAddr := &mockAddressable{name: "name2", typ: "type2", conditionResult: false} - psm.Put("key1", trueAddr) - psm.Put("key2", falseAddr) - - v1, _ := psm.Get("key1") - if !v1.ConditionIsValid("lhs", "rhs") { - t.Error("expected ConditionIsValid to return true") - } - - v2, _ := psm.Get("key2") - if v2.ConditionIsValid("lhs", "rhs") { - t.Error("expected ConditionIsValid to return false") - } -} - -func TestParameterSuffixMapGetType(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("name1", "typeA")) - psm.Put("key2", newMock("name2", "typeB")) - - v1, _ := psm.Get("key1") - if v1.GetType() != "typeA" { - t.Errorf("expected typeA, got %s", v1.GetType()) - } - - v2, _ := psm.Get("key2") - if v2.GetType() != "typeB" { - t.Errorf("expected typeB, got %s", v2.GetType()) - } -} - -func TestParameterSuffixMapGetName(t *testing.T) { - psm := suffix.NewParameterSuffixMap() - psm.Put("key1", newMock("nameA", "type1")) - psm.Put("key2", newMock("nameB", "type2")) - - v1, _ := psm.Get("key1") - if v1.GetName() != "nameA" { - t.Errorf("expected nameA, got %s", v1.GetName()) - } - - v2, _ := psm.Get("key2") - if v2.GetName() != "nameB" { - t.Errorf("expected nameB, got %s", v2.GetName()) - } -} diff --git a/internal/stackql/symtab/symtab.go b/internal/stackql/symtab/symtab.go index ad56ab6da..0281422cf 100644 --- a/internal/stackql/symtab/symtab.go +++ b/internal/stackql/symtab/symtab.go @@ -5,8 +5,8 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/go-suffix-map/pkg/suffixmap" + "github.com/stackql/stackql/internal/stackql/logging" ) type Entry struct { diff --git a/internal/stackql/tableinsertioncontainer/table_insertion_container.go b/internal/stackql/tableinsertioncontainer/table_insertion_container.go index e36c4bf40..963fad097 100644 --- a/internal/stackql/tableinsertioncontainer/table_insertion_container.go +++ b/internal/stackql/tableinsertioncontainer/table_insertion_container.go @@ -1,8 +1,8 @@ package tableinsertioncontainer import ( - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/pkg/txncounter" ) diff --git a/internal/stackql/tablemetadata/const.go b/internal/stackql/tablemetadata/const.go deleted file mode 100644 index caf57cc6e..000000000 --- a/internal/stackql/tablemetadata/const.go +++ /dev/null @@ -1,5 +0,0 @@ -package tablemetadata - -const ( - defaultSelectItemsKey = "items" -) diff --git a/internal/stackql/tablemetadata/extended_table_metadata.go b/internal/stackql/tablemetadata/extended_table_metadata.go index 751a252d2..69c33f680 100644 --- a/internal/stackql/tablemetadata/extended_table_metadata.go +++ b/internal/stackql/tablemetadata/extended_table_metadata.go @@ -3,11 +3,9 @@ package tablemetadata import ( "fmt" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/provider" ) var ( @@ -16,46 +14,36 @@ var ( type ExtendedTableMetadata interface { GetAlias() string - GetGraphQL() (formulation.GraphQL, bool) + GetHeirarchyObjects() HeirarchyObjects - GetHTTPArmoury() (formulation.HTTPArmoury, error) GetInputTableName() (string, error) - GetMethod() (formulation.StandardOperationStore, error) + GetMethodStr() (string, error) - GetProvider() (provider.IProvider, error) + GetProviderStr() (string, error) - GetProviderObject() (formulation.Provider, error) + GetQueryUniqueID() string - GetRequestSchema() (formulation.Schema, error) - GetOptionalParameters() map[string]formulation.Addressable - GetRequiredParameters() map[string]formulation.Addressable - GetParametersIncludingNativeCasing() map[string]formulation.Addressable - GetResource() (formulation.Resource, error) + GetResourceStr() (string, error) GetResponseSchemaStr() (string, error) - GetResponseSchemaAndMediaType() (formulation.Schema, string, error) - GetSelectableObjectSchema() (formulation.Schema, error) + GetSelectItemsKey() string - GetSelectSchemaAndObjectPath() (formulation.Schema, string, error) - GetService() (formulation.Service, error) + GetServiceStr() (string, error) GetSQLDataSource() (sql_datasource.SQLDataSource, bool) GetStackQLTableName() (string, error) - GetTableFilter() func(formulation.ITable) (formulation.ITable, error) + GetTableName() (string, error) GetUniqueID() string IsLocallyExecutable() bool - IsSimple() bool + GetIndirect() (astindirect.Indirect, bool) GetView() (internaldto.RelationDTO, bool) GetSubquery() (internaldto.SubqueryDTO, bool) - LookupSelectItemsKey() string + SetSelectItemsKey(string) SetSQLDataSource(sql_datasource.SQLDataSource) - SetTableFilter(f func(formulation.ITable) (formulation.ITable, error)) - SetPushdownIntent(intent formulation.PushdownIntent) - GetPushdownIntent() (formulation.PushdownIntent, bool) - WithGetHTTPArmoury(f func() (formulation.HTTPArmoury, error)) ExtendedTableMetadata + WithIndirect(astindirect.Indirect) ExtendedTableMetadata WithResponseSchemaStr(rss string) (ExtendedTableMetadata, error) IsPGInternalObject() bool @@ -68,34 +56,28 @@ type ExtendedTableMetadata interface { } type standardExtendedTableMetadata struct { - tableFilter func(formulation.ITable) (formulation.ITable, error) colsVisited map[string]bool heirarchyObjects HeirarchyObjects isLocallyExecutable bool - getHTTPArmoury func() (formulation.HTTPArmoury, error) selectItemsKey string alias string inputTableName string indirect astindirect.Indirect sqlDataSource sql_datasource.SQLDataSource isOnClauseHoistable bool - pushdownIntent formulation.PushdownIntent } func (ex *standardExtendedTableMetadata) Clone() ExtendedTableMetadata { return &standardExtendedTableMetadata{ - tableFilter: ex.tableFilter, colsVisited: ex.colsVisited, heirarchyObjects: ex.heirarchyObjects, isLocallyExecutable: ex.isLocallyExecutable, - getHTTPArmoury: ex.getHTTPArmoury, selectItemsKey: ex.selectItemsKey, alias: ex.alias, inputTableName: ex.inputTableName, indirect: ex.indirect, sqlDataSource: ex.sqlDataSource, isOnClauseHoistable: ex.isOnClauseHoistable, - pushdownIntent: ex.pushdownIntent, } } @@ -145,18 +127,6 @@ func (ex *standardExtendedTableMetadata) IsMaterializedView() bool { return ex.heirarchyObjects.GetHeirarchyIDs().IsMaterializedView() } -// SetPushdownIntent records the neutral query-option push-down intent the analysis phase -// resolved for this acquire. It is handed to the HTTP preparator (HTTPPreparator. -// WithPushdownIntent) when the request is prepared, so any-sdk owns the dialect translation -// and request application and the executor stays protocol-agnostic. -func (ex *standardExtendedTableMetadata) SetPushdownIntent(intent formulation.PushdownIntent) { - ex.pushdownIntent = intent -} - -func (ex *standardExtendedTableMetadata) GetPushdownIntent() (formulation.PushdownIntent, bool) { - return ex.pushdownIntent, ex.pushdownIntent != nil -} - func (ex *standardExtendedTableMetadata) SetIsOnClauseHoistable(isOnClauseHoistable bool) { ex.isOnClauseHoistable = isOnClauseHoistable } @@ -202,88 +172,10 @@ func (ex *standardExtendedTableMetadata) SetSelectItemsKey(k string) { ex.selectItemsKey = k } -func (ex *standardExtendedTableMetadata) SetTableFilter(f func(formulation.ITable) (formulation.ITable, error)) { - ex.tableFilter = f -} - -func (ex *standardExtendedTableMetadata) GetTableFilter() func(formulation.ITable) (formulation.ITable, error) { - return ex.tableFilter -} - -func (ex *standardExtendedTableMetadata) GetGraphQL() (formulation.GraphQL, bool) { - //nolint:lll // defer fix - if ex.heirarchyObjects.GetMethod() != nil && ex.heirarchyObjects.GetMethod().GetGraphQL() != nil && !ex.heirarchyObjects.GetMethod().GetGraphQL().IsEmpty() { - return ex.heirarchyObjects.GetMethod().GetGraphQL(), true - } - return nil, false -} - -func (ex *standardExtendedTableMetadata) GetRequiredParameters() map[string]formulation.Addressable { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetMethod() == nil { - return nil - } - rv := map[string]formulation.Addressable{} - for k, v := range ex.heirarchyObjects.GetMethod().GetRequiredParameters() { - rv[k] = v - } - return rv -} - -func (ex *standardExtendedTableMetadata) GetOptionalParameters() map[string]formulation.Addressable { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetMethod() == nil { - return nil - } - rv := map[string]formulation.Addressable{} - for k, v := range ex.heirarchyObjects.GetMethod().GetOptionalParameters() { - rv[k] = v - } - return rv -} - -// GetParametersIncludingNativeCasing returns the method parameters keyed by their wire name -// and, when the method declares request native casing, also by their snake_case alias (the -// same Addressable; its GetName is the wire name). any-sdk owns the casing translation; -// stackql uses this to resolve snake_case WHERE / INSERT keys to their wire parameter. -func (ex *standardExtendedTableMetadata) GetParametersIncludingNativeCasing() map[string]formulation.Addressable { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetMethod() == nil { - return nil - } - rv := map[string]formulation.Addressable{} - for k, v := range ex.heirarchyObjects.GetMethod().GetParametersIncludingNativeCasing() { - rv[k] = v - } - return rv -} - -func (ex *standardExtendedTableMetadata) GetHTTPArmoury() (formulation.HTTPArmoury, error) { - if ex.getHTTPArmoury == nil { - return nil, fmt.Errorf("nil getHttpAroury() function in ExtendedTableMetadata object") - } - return ex.getHTTPArmoury() -} - -func (ex *standardExtendedTableMetadata) WithGetHTTPArmoury( - f func() (formulation.HTTPArmoury, error), -) ExtendedTableMetadata { - ex.getHTTPArmoury = f - return ex -} - -func (ex *standardExtendedTableMetadata) LookupSelectItemsKey() string { - if ex.heirarchyObjects == nil { - return defaultSelectItemsKey - } - return ex.heirarchyObjects.LookupSelectItemsKey() -} - func (ex *standardExtendedTableMetadata) GetAlias() string { return ex.alias } -func (ex *standardExtendedTableMetadata) IsSimple() bool { - return ex.isSimple() -} - func (ex *standardExtendedTableMetadata) GetSubquery() (internaldto.SubqueryDTO, bool) { return ex.heirarchyObjects.GetSubquery() } @@ -292,11 +184,6 @@ func (ex *standardExtendedTableMetadata) GetView() (internaldto.RelationDTO, boo return ex.heirarchyObjects.GetView() } -func (ex *standardExtendedTableMetadata) isSimple() bool { - //nolint:lll // complex boolean - return ex.heirarchyObjects != nil && ((ex.heirarchyObjects.GetMethodSet() != nil && ex.heirarchyObjects.GetMethodSet().Size() > 0) || ex.heirarchyObjects.GetMethod() != nil) -} - func (ex *standardExtendedTableMetadata) GetUniqueID() string { if ex.alias != "" { return ex.alias @@ -311,60 +198,6 @@ func (ex *standardExtendedTableMetadata) GetQueryUniqueID() string { return ex.heirarchyObjects.GetTableName() } -func (ex *standardExtendedTableMetadata) GetProvider() (provider.IProvider, error) { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetProvider() == nil { - return nil, fmt.Errorf("cannot resolve Provider") - } - return ex.heirarchyObjects.GetProvider(), nil -} - -func (ex *standardExtendedTableMetadata) GetProviderObject() (formulation.Provider, error) { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetProvider() == nil { - return nil, fmt.Errorf("cannot resolve Provider object") - } - return ex.heirarchyObjects.GetProvider().GetProvider() -} - -func (ex *standardExtendedTableMetadata) GetService() (formulation.Service, error) { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetServiceHdl() == nil { - return nil, fmt.Errorf("cannot resolve ServiceHandle") - } - return ex.heirarchyObjects.GetServiceHdl(), nil -} - -func (ex *standardExtendedTableMetadata) GetResource() (formulation.Resource, error) { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetResource() == nil { - return nil, fmt.Errorf("cannot resolve Resource") - } - return ex.heirarchyObjects.GetResource(), nil -} - -func (ex *standardExtendedTableMetadata) GetMethod() (formulation.StandardOperationStore, error) { - return ex.getMethod() -} - -func (ex *standardExtendedTableMetadata) getMethod() (formulation.StandardOperationStore, error) { - if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetMethod() == nil { - return nil, fmt.Errorf("cannot resolve Method") - } - return ex.heirarchyObjects.GetMethod(), nil -} - -func (ex *standardExtendedTableMetadata) GetSelectSchemaAndObjectPath() (formulation.Schema, string, error) { - return ex.heirarchyObjects.GetSelectSchemaAndObjectPath() -} - -func (ex *standardExtendedTableMetadata) GetResponseSchemaAndMediaType() (formulation.Schema, string, error) { - if ex.isSimple() { - return ex.heirarchyObjects.GetResponseSchemaAndMediaType() - } - return nil, "", fmt.Errorf("error extracting response schema and media type: views not yet supported") -} - -func (ex *standardExtendedTableMetadata) GetRequestSchema() (formulation.Schema, error) { - return ex.heirarchyObjects.GetRequestSchema() -} - func (ex *standardExtendedTableMetadata) GetServiceStr() (string, error) { if ex.heirarchyObjects == nil || ex.heirarchyObjects.GetHeirarchyIDs().GetServiceStr() == "" { return "", fmt.Errorf("cannot resolve ServiceStr") @@ -426,10 +259,6 @@ func (ex *standardExtendedTableMetadata) GetInputTableName() (string, error) { return ex.inputTableName, nil } -func (ex *standardExtendedTableMetadata) GetSelectableObjectSchema() (formulation.Schema, error) { - return ex.heirarchyObjects.GetSelectableObjectSchema() -} - func NewExtendedTableMetadata(heirarchyObjects HeirarchyObjects, tableName string, alias string) ExtendedTableMetadata { return &standardExtendedTableMetadata{ colsVisited: make(map[string]bool), diff --git a/internal/stackql/tablemetadata/hierarchy_objects.go b/internal/stackql/tablemetadata/hierarchy_objects.go index ddd6cce70..1bff7db61 100644 --- a/internal/stackql/tablemetadata/hierarchy_objects.go +++ b/internal/stackql/tablemetadata/hierarchy_objects.go @@ -1,12 +1,8 @@ package tablemetadata import ( - "fmt" - - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/provider" ) var ( @@ -15,29 +11,15 @@ var ( type HeirarchyObjects interface { GetHeirarchyIDs() internaldto.HeirarchyIdentifiers - GetObjectSchema() (formulation.Schema, error) - GetProvider() provider.IProvider - GetRequestSchema() (formulation.Schema, error) - GetResponseSchemaAndMediaType() (formulation.Schema, string, error) - GetSelectableObjectSchema() (formulation.Schema, error) - GetSelectSchemaAndObjectPath() (formulation.Schema, string, error) + GetSQLDataSource() (sql_datasource.SQLDataSource, bool) GetTableName() string GetSubquery() (internaldto.SubqueryDTO, bool) GetView() (internaldto.RelationDTO, bool) - LookupSelectItemsKey() string - SetProvider(provider.IProvider) + SetSQLDataSource(sql_datasource.SQLDataSource) // De facto inheritance - GetServiceHdl() formulation.Service - GetResource() formulation.Resource - GetMethodSet() formulation.MethodSet - GetMethod() formulation.StandardOperationStore - SetMethod(formulation.StandardOperationStore) - SetMethodSet(formulation.MethodSet) - SetMethodStr(string) - SetResource(formulation.Resource) - SetServiceHdl(formulation.Service) + IsPGInternalObject() bool SetIndirect(internaldto.RelationDTO) GetIndirect() (internaldto.RelationDTO, bool) @@ -47,15 +29,12 @@ type HeirarchyObjects interface { func NewHeirarchyObjects(hIDs internaldto.HeirarchyIdentifiers, isAwait bool) HeirarchyObjects { return &standardHeirarchyObjects{ heirarchyIDs: hIDs, - hr: internaldto.NewHeirarchy(hIDs), isAwait: isAwait, } } type standardHeirarchyObjects struct { - hr internaldto.Heirarchy heirarchyIDs internaldto.HeirarchyIdentifiers - prov provider.IProvider sqlDataSource sql_datasource.SQLDataSource indirect internaldto.RelationDTO isAwait bool @@ -77,10 +56,6 @@ func (ho *standardHeirarchyObjects) IsPGInternalObject() bool { return ho.heirarchyIDs.IsPgInternalObject() } -func (ho *standardHeirarchyObjects) GetServiceHdl() formulation.Service { - return ho.hr.GetServiceHdl() -} - func (ho *standardHeirarchyObjects) GetSQLDataSource() (sql_datasource.SQLDataSource, bool) { return ho.sqlDataSource, ho.sqlDataSource != nil } @@ -97,127 +72,10 @@ func (ho *standardHeirarchyObjects) GetSubquery() (internaldto.SubqueryDTO, bool return ho.heirarchyIDs.GetSubquery() } -func (ho *standardHeirarchyObjects) GetResource() formulation.Resource { - return ho.hr.GetResource() -} - -func (ho *standardHeirarchyObjects) GetMethodSet() formulation.MethodSet { - return ho.hr.GetMethodSet() -} - -func (ho *standardHeirarchyObjects) GetMethod() formulation.StandardOperationStore { - return ho.hr.GetMethod() -} - -func (ho *standardHeirarchyObjects) SetServiceHdl(sh formulation.Service) { - ho.hr.SetServiceHdl(sh) -} - -func (ho *standardHeirarchyObjects) SetResource(r formulation.Resource) { - ho.hr.SetResource(r) -} - -func (ho *standardHeirarchyObjects) SetMethodSet(mSet formulation.MethodSet) { - ho.hr.SetMethodSet(mSet) -} - -func (ho *standardHeirarchyObjects) SetMethod(m formulation.StandardOperationStore) { - ho.hr.SetMethod(m) -} - -func (ho *standardHeirarchyObjects) SetProvider(prov provider.IProvider) { - ho.prov = prov -} - -func (ho *standardHeirarchyObjects) GetProvider() provider.IProvider { - return ho.prov -} - func (ho *standardHeirarchyObjects) GetHeirarchyIDs() internaldto.HeirarchyIdentifiers { return ho.heirarchyIDs } -func (ho *standardHeirarchyObjects) SetMethodStr(mStr string) { - ho.hr.SetMethodStr(mStr) -} - -func (ho *standardHeirarchyObjects) LookupSelectItemsKey() string { - method := ho.GetMethod() - return lookupSelectItemsKey(method) -} - -func LookupSelectItemsKey(method formulation.OperationStore) string { - return lookupSelectItemsKey(method) -} - -func lookupSelectItemsKey(method formulation.OperationStore) string { - if method == nil { - return defaultSelectItemsKey - } - if sk := method.GetSelectItemsKey(); sk != "" { - return sk - } - responseSchema, _, err := method.GetResponseBodySchemaAndMediaType() - if responseSchema == nil || err != nil { - return "" - } - switch responseSchema.GetType() { - case "string", "integer": - return formulation.AnonymousColumnName - } - return defaultSelectItemsKey -} - -func (ho *standardHeirarchyObjects) GetResponseSchemaAndMediaType() (formulation.Schema, string, error) { - m := ho.GetMethod() - if m == nil { - return nil, "", fmt.Errorf("method is nil") - } - return m.GetResponseBodySchemaAndMediaType() -} - -func (ho *standardHeirarchyObjects) GetSelectSchemaAndObjectPath() (formulation.Schema, string, error) { - m := ho.GetMethod() - if m == nil { - return nil, "", fmt.Errorf("method is nil") - } - return m.GetSelectSchemaAndObjectPath() -} - -func (ho *standardHeirarchyObjects) GetRequestSchema() (formulation.Schema, error) { - m := ho.GetMethod() - if m == nil { - return nil, fmt.Errorf("method is nil") - } - return ho.GetRequestSchema() -} - func (ho *standardHeirarchyObjects) GetTableName() string { return ho.heirarchyIDs.GetTableName() } - -func (ho *standardHeirarchyObjects) GetObjectSchema() (formulation.Schema, error) { - return ho.getObjectSchema() -} - -func (ho *standardHeirarchyObjects) getObjectSchema() (formulation.Schema, error) { - rv, _, err := ho.GetMethod().GetResponseBodySchemaAndMediaType() - return rv, err -} - -func (ho *standardHeirarchyObjects) GetSelectableObjectSchema() (formulation.Schema, error) { - unsuitableSchemaMsg := "GetSelectableObjectSchema(): schema unsuitable for select query" - itemObjS, _, err := ho.GetMethod().GetSelectSchemaAndObjectPath() - if err != nil { - return nil, fmt.Errorf("%w: %s", err, unsuitableSchemaMsg) - } - if itemObjS == nil { - m, ok := ho.GetMethod().GetResponse() - ts := "" - if ok { - ts = fmt.Sprintf("'%T'", m.GetObjectKey()) - } - return nil, fmt.Errorf("could not locate dml object for response type %s", ts) - } - return itemObjS, nil -} diff --git a/internal/stackql/tablenamespace/builder.go b/internal/stackql/tablenamespace/builder.go index e84a7d781..2c44610c1 100644 --- a/internal/stackql/tablenamespace/builder.go +++ b/internal/stackql/tablenamespace/builder.go @@ -4,7 +4,7 @@ import ( "regexp" "text/template" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/templatenamespace" ) diff --git a/internal/stackql/tablenamespace/collection.go b/internal/stackql/tablenamespace/collection.go index 31586e432..8fc61f883 100644 --- a/internal/stackql/tablenamespace/collection.go +++ b/internal/stackql/tablenamespace/collection.go @@ -1,9 +1,9 @@ package tablenamespace import ( - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" ) type Collection interface { diff --git a/internal/stackql/tablenamespace/director.go b/internal/stackql/tablenamespace/director.go index fc63f5a58..b3bbd5204 100644 --- a/internal/stackql/tablenamespace/director.go +++ b/internal/stackql/tablenamespace/director.go @@ -4,9 +4,9 @@ import ( "regexp" "text/template" - "github.com/stackql/any-sdk/pkg/constants" - "github.com/stackql/any-sdk/pkg/dto" - "github.com/stackql/any-sdk/public/sqlengine" + "github.com/stackql/stackql/internal/stackql/constants" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/pkg/textutil" ) diff --git a/internal/stackql/tablenamespace/table_namespace.go b/internal/stackql/tablenamespace/table_namespace.go index 9f17d67d1..3eceaf60b 100644 --- a/internal/stackql/tablenamespace/table_namespace.go +++ b/internal/stackql/tablenamespace/table_namespace.go @@ -6,9 +6,9 @@ import ( "strings" "time" - "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/sql_system" + "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/templatenamespace" ) diff --git a/internal/stackql/taxonomy/annotated_tabulation_map.go b/internal/stackql/taxonomy/annotated_tabulation_map.go deleted file mode 100644 index 16dd738e5..000000000 --- a/internal/stackql/taxonomy/annotated_tabulation_map.go +++ /dev/null @@ -1,8 +0,0 @@ -package taxonomy - -import ( - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/util" -) - -type AnnotatedTabulationMap map[sqlparser.SQLNode]util.AnnotatedTabulation diff --git a/internal/stackql/taxonomy/annotation_context.go b/internal/stackql/taxonomy/annotation_context.go deleted file mode 100644 index 67ae2f188..000000000 --- a/internal/stackql/taxonomy/annotation_context.go +++ /dev/null @@ -1,252 +0,0 @@ -package taxonomy - -import ( - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/streaming" - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/parserutil" - "github.com/stackql/stackql/internal/stackql/tablemetadata" - "github.com/stackql/stackql/internal/stackql/util" -) - -// TODO: -// - For views, need API to get child. -type AnnotationCtx interface { - GetHIDs() internaldto.HeirarchyIdentifiers - IsDynamic() bool - GetView() (internaldto.RelationDTO, bool) - GetSubquery() (internaldto.SubqueryDTO, bool) - GetInputTableName() (string, error) - GetParameters() map[string]interface{} - GetSchema() formulation.Schema - GetTableMeta() tablemetadata.ExtendedTableMetadata - Prepare(handlerCtx handler.HandlerContext, inStream streaming.MapStream) error - SetDynamic() - IsAwait() bool - Equals(AnnotationCtx) bool - Clone() AnnotationCtx -} - -type standardAnnotationCtx struct { - isDynamic bool - schema formulation.Schema - hIDs internaldto.HeirarchyIdentifiers - tableMeta tablemetadata.ExtendedTableMetadata - parameters map[string]interface{} - isAwait bool -} - -func NewStaticStandardAnnotationCtx( - schema formulation.Schema, - hIDs internaldto.HeirarchyIdentifiers, - tableMeta tablemetadata.ExtendedTableMetadata, - parameters map[string]interface{}, - isAwait bool, -) AnnotationCtx { - return &standardAnnotationCtx{ - isDynamic: false, - schema: schema, - hIDs: hIDs, - tableMeta: tableMeta, - parameters: parameters, - isAwait: isAwait, - } -} - -func (ac *standardAnnotationCtx) IsAwait() bool { - return ac.isAwait -} - -//nolint:gocognit // acceptable -func (ac *standardAnnotationCtx) Equals(other AnnotationCtx) bool { - otherStandard, isStandard := other.(*standardAnnotationCtx) - if !isStandard { - return false - } - if ac.isDynamic != otherStandard.isDynamic { - return false - } - if ac.isAwait != otherStandard.isAwait { - return false - } - if ac.schema != otherStandard.schema { - return false - } - if ac.hIDs != otherStandard.hIDs { - return false - } - if !ac.tableMeta.Equals(otherStandard.tableMeta) { - return false - } - if len(ac.parameters) != len(otherStandard.parameters) { - return false - } - for k, v := range ac.parameters { - rhs, paramExists := otherStandard.parameters[k] - if rhs != v { - if paramExists { - logging.GetLogger().Debugf("param %s not equal: %v != %v\n", k, v, rhs) - } - lhsStdParam, lhsOk := v.(parserutil.ParameterMetadata) - rhsStdParam, rhsOk := rhs.(parserutil.ParameterMetadata) - if lhsOk && rhsOk { - if !lhsStdParam.Equals(rhsStdParam) { - return true - } - } - return false - } - } - return true -} - -func (ac *standardAnnotationCtx) Clone() AnnotationCtx { - clonedParams := make(map[string]interface{}) - for k, v := range ac.parameters { - clonedParams[k] = v - } - return &standardAnnotationCtx{ - isDynamic: ac.isDynamic, - schema: ac.schema, - hIDs: ac.hIDs, - tableMeta: ac.tableMeta.Clone(), - parameters: clonedParams, - isAwait: ac.isAwait, - } -} - -func (ac *standardAnnotationCtx) IsDynamic() bool { - return ac.isDynamic -} - -func (ac *standardAnnotationCtx) GetView() (internaldto.RelationDTO, bool) { - return ac.hIDs.GetView() -} - -func (ac *standardAnnotationCtx) GetSubquery() (internaldto.SubqueryDTO, bool) { - return ac.hIDs.GetSubquery() -} - -func (ac *standardAnnotationCtx) SetDynamic() { - ac.isDynamic = true -} - -func (ac *standardAnnotationCtx) Prepare( - handlerCtx handler.HandlerContext, //nolint:revive // future proofing - stream streaming.MapStream, -) error { - // TODO: accomodate SQL data source - sqlDataSource, isSQLDataSource := ac.GetTableMeta().GetSQLDataSource() - if isSQLDataSource { - ac.tableMeta.SetSQLDataSource(sqlDataSource) - // TODO: persist mirror table here a la GenerateInsertDML() - return nil - } - pr, err := ac.GetTableMeta().GetProvider() - if err != nil { - return err - } - svc, err := ac.GetTableMeta().GetService() - if err != nil { - return err - } - opStore, err := ac.GetTableMeta().GetMethod() - if err != nil { - return err - } - params := ac.GetParameters() - // LAZY EVAL if dynamic - if ac.isDynamic { - viewDTO, isView := ac.GetView() - // TODO: fill this out - if isView { - logging.GetLogger().Debugf("viewDTO = %v\n", viewDTO) - } - prov, provErr := pr.GetProvider() - if provErr != nil { - return provErr - } - ac.tableMeta.WithGetHTTPArmoury( - func() (formulation.HTTPArmoury, error) { - httpPreparator := formulation.NewHTTPPreparator( - prov, - svc, - opStore, - nil, - stream, - nil, - logging.GetLogger(), - ) - httpArmoury, armouryErr := httpPreparator.BuildHTTPRequestCtx( - formulation.NewHTTPPreparatorConfig(true), - ) - return httpArmoury, armouryErr - }, - ) - return nil - } - - ac.tableMeta.WithGetHTTPArmoury( - func() (formulation.HTTPArmoury, error) { - // need to dynamically generate stream, otherwise repeated calls result in empty body - parametersCleaned, cleanErr := util.TransformSQLRawParameters(params, true) - if cleanErr != nil { - return nil, cleanErr - } - stream.Write( //nolint:errcheck // TODO: handle error - []map[string]interface{}{ - parametersCleaned, - }, - ) - prov, provErr := pr.GetProvider() - if provErr != nil { - return nil, provErr - } - httpPreparator := formulation.NewHTTPPreparator( - prov, - svc, - opStore, - nil, - stream, - nil, - logging.GetLogger(), - ) - // Apply the analysis-phase query-option push-down: hand the neutral intent to the - // preparator, which (inside any-sdk) translates it to the method's dialect and sets - // the request query params. The HTTPArmoury stays opaque to stackql. - if intent, intentOk := ac.tableMeta.GetPushdownIntent(); intentOk { - httpPreparator = httpPreparator.WithPushdownIntent(intent) - } - httpArmoury, armouryErr := httpPreparator.BuildHTTPRequestCtx( - formulation.NewHTTPPreparatorConfig(true), - ) - if armouryErr != nil { - return nil, armouryErr - } - return httpArmoury, nil - }, - ) - return nil -} - -func (ac *standardAnnotationCtx) GetHIDs() internaldto.HeirarchyIdentifiers { - return ac.hIDs -} - -func (ac *standardAnnotationCtx) GetParameters() map[string]interface{} { - return ac.parameters -} - -func (ac *standardAnnotationCtx) GetSchema() formulation.Schema { - return ac.schema -} - -func (ac *standardAnnotationCtx) GetInputTableName() (string, error) { - return ac.tableMeta.GetInputTableName() -} - -func (ac *standardAnnotationCtx) GetTableMeta() tablemetadata.ExtendedTableMetadata { - return ac.tableMeta -} diff --git a/internal/stackql/taxonomy/annotation_context_map.go b/internal/stackql/taxonomy/annotation_context_map.go deleted file mode 100644 index a2db3d1ed..000000000 --- a/internal/stackql/taxonomy/annotation_context_map.go +++ /dev/null @@ -1,44 +0,0 @@ -package taxonomy - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" -) - -type AnnotationCtxMap map[sqlparser.SQLNode]AnnotationCtx - -func (am AnnotationCtxMap) AssignParams() error { - rv := make(map[string]interface{}) - for k, v := range am { - for p, pVal := range v.GetParameters() { - switch pVal.(type) { //nolint:gocritic // low impact - case *sqlparser.ColName: - } - aliasedName, ok := k.(*sqlparser.AliasedTableExpr) - if !ok { - continue - } - tableAlias := aliasedName.As.GetRawVal() - nk := fmt.Sprintf("%s.%s", tableAlias, p) - rv[nk] = pVal - } - } - return nil -} - -func (am AnnotationCtxMap) GetStringParams() map[string]interface{} { - rv := make(map[string]interface{}) - for k, v := range am { - for p, pVal := range v.GetParameters() { - aliasedName, ok := k.(*sqlparser.AliasedTableExpr) - if !ok { - continue - } - tableAlias := aliasedName.As.GetRawVal() - nk := fmt.Sprintf("%s.%s", tableAlias, p) - rv[nk] = pVal - } - } - return rv -} diff --git a/internal/stackql/taxonomy/annotation_context_split_map.go b/internal/stackql/taxonomy/annotation_context_split_map.go deleted file mode 100644 index 28ca242ff..000000000 --- a/internal/stackql/taxonomy/annotation_context_split_map.go +++ /dev/null @@ -1,35 +0,0 @@ -package taxonomy - -type AnnotationCtxSplitMap interface { - Put(AnnotationCtx, AnnotationCtx) - Get(AnnotationCtx) ([]AnnotationCtx, bool) - Len() int -} - -type annotationCtxSplitMap struct { - m map[AnnotationCtx][]AnnotationCtx -} - -func NewAnnotationCtxSplitMap() AnnotationCtxSplitMap { - return &annotationCtxSplitMap{ - m: make(map[AnnotationCtx][]AnnotationCtx), - } -} - -func (am *annotationCtxSplitMap) Put(k, v AnnotationCtx) { - _, ok := am.m[k] - if ok { - am.m[k] = append(am.m[k], v) - return - } - am.m[k] = []AnnotationCtx{v} -} - -func (am *annotationCtxSplitMap) Get(k AnnotationCtx) ([]AnnotationCtx, bool) { - rv, ok := am.m[k] - return rv, ok -} - -func (am *annotationCtxSplitMap) Len() int { - return len(am.m) -} diff --git a/internal/stackql/taxonomy/hierarchy.go b/internal/stackql/taxonomy/hierarchy.go index c4f3b47fd..f6fbf91fc 100644 --- a/internal/stackql/taxonomy/hierarchy.go +++ b/internal/stackql/taxonomy/hierarchy.go @@ -3,9 +3,6 @@ package taxonomy import ( "fmt" - "github.com/stackql/any-sdk/pkg/logging" - "github.com/stackql/any-sdk/pkg/name_mangle" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/parserutil" @@ -217,7 +214,6 @@ func GetHeirarchyFromStatement( isAwait bool, ) (tablemetadata.HeirarchyObjects, error) { var hIDs internaldto.HeirarchyIdentifiers - getFirstAvailableMethod := false if parameters == nil { parameters = parserutil.NewParameterMap() } @@ -225,42 +221,6 @@ func GetHeirarchyFromStatement( if err != nil { return nil, err } - methodRequired := true - var methodAction string - switch n := node.(type) { - case *sqlparser.Exec, *sqlparser.ExecSubquery: - case *sqlparser.Select: - methodAction = "select" - case *sqlparser.DescribeTable: - case *sqlparser.DescribeMethod: - return nil, fmt.Errorf("DESCRIBE METHOD is dispatched outside of hierarchy resolution") - case sqlparser.TableName: - case *sqlparser.AliasedTableExpr: - switch n.Expr.(type) { //nolint:gocritic // this is expressive enough - case *sqlparser.Subquery: - retVal := tablemetadata.NewHeirarchyObjects(hIDs, isAwait) - return retVal, nil - } - return GetHeirarchyFromStatement(handlerCtx, n.Expr, parameters, viewPermissive, isAwait) - case *sqlparser.Show: - switch strings.ToUpper(n.Type) { - case "INSERT": - methodAction = "insert" - getFirstAvailableMethod = true - case "METHODS": - methodRequired = false - default: - return nil, fmt.Errorf("cannot resolve taxonomy for SHOW statement of type = '%s'", strings.ToUpper(n.Type)) - } - case *sqlparser.Insert: - methodAction = "insert" - case *sqlparser.Delete: - methodAction = "delete" - case *sqlparser.Update: - methodAction = strings.ToLower(n.Action) - default: - return nil, fmt.Errorf("cannot resolve taxonomy") - } retVal := tablemetadata.NewHeirarchyObjects(hIDs, isAwait) sqlDataSource, isSQLDataSource := handlerCtx.GetSQLDataSource(hIDs.GetProviderStr()) if isSQLDataSource { @@ -272,139 +232,9 @@ func GetHeirarchyFromStatement( if isPgInternal { return retVal, nil } - prov, err := handlerCtx.GetProvider(hIDs.GetProviderStr()) - retVal.SetProvider(prov) - viewDTO, viewExists := retVal.GetView() - var meth formulation.StandardOperationStore - var methStr string - var methodErr error - if methodAction == "" { - methodAction = "select" - } - if viewExists { - retVal.SetIndirect(viewDTO) - logging.GetLogger().Debugf("viewDTO = %v\n", viewDTO) - // return retVal, nil //nolint:nilerr // acceptable - } - if err != nil { - return returnViewOnErrorIfPresent(retVal, err, viewExists) - } - svcHdl, err := prov.GetServiceShard(hIDs.GetServiceStr(), hIDs.GetResourceStr(), handlerCtx.GetRuntimeContext()) - if err != nil { - return returnViewOnErrorIfPresent(retVal, err, viewExists) - } - retVal.SetServiceHdl(svcHdl) - rsc, err := prov.GetResource(hIDs.GetServiceStr(), hIDs.GetResourceStr(), handlerCtx.GetRuntimeContext()) - if err != nil { - return returnViewOnErrorIfPresent(retVal, err, viewExists) - } - retVal.SetResource(rsc) - viewNameMangler := name_mangle.NewViewNameMangler() - //nolint:nestif // not overly complex - if viewCollection, ok := rsc.GetViewsForSqlDialect( - handlerCtx.GetSQLSystem().GetName()); ok && methodAction == "select" && !viewExists { - for i, view := range viewCollection { - viewNameNaive := view.GetNameNaive() - viewName := viewNameMangler.MangleName(viewNameNaive, i) - // TODO: mutex required or some other strategy - viewDTO, viewExists = handlerCtx.GetSQLSystem().GetViewByNameAndParameters( - viewName, parameters.GetStringified()) - if !viewExists { - // TODO: resolve any possible data race - err = handlerCtx.GetSQLSystem().CreateView(viewName, view.GetDDL(), true, nil) - if err != nil { - return nil, err - } - params := parameters.GetStringified() - viewDTO, viewExists = handlerCtx.GetSQLSystem().GetViewByNameAndParameters( - hIDs.GetTableName(), params) - if viewPermissive { - viewDTO, viewExists = handlerCtx.GetSQLSystem().GetViewByName(hIDs.GetTableName()) - } - if viewExists { - retVal.SetIndirect(viewDTO) - } - return retVal, nil - } - retVal.SetIndirect(viewDTO) - return retVal, nil //nolint:staticcheck // TODO: fix this - } - } - var method formulation.StandardOperationStore - switch node.(type) { - case *sqlparser.Exec, *sqlparser.ExecSubquery: - method, err = rsc.FindMethod(hIDs.GetMethodStr()) - if err != nil { - return returnViewOnErrorIfPresent(retVal, err, viewExists) - } - retVal.SetMethod(method) + if view, ok := hIDs.GetView(); ok { + retVal.SetIndirect(view) return retVal, nil } - //nolint:nestif,ineffassign // acceptable for now - if methodRequired { - switch node.(type) { //nolint:gocritic // this is expressive enough - case *sqlparser.DescribeTable: - if viewExists { - return retVal, nil - } - m, mStr, mErr := prov.InferDescribeMethod(rsc) - if mErr != nil { - return nil, mErr - } - retVal.SetMethod(m) - retVal.SetMethodStr(mStr) - return retVal, nil - case *sqlparser.DescribeMethod: - return nil, fmt.Errorf("DESCRIBE METHOD is dispatched outside of hierarchy resolution") - } - if getFirstAvailableMethod { - meth, methStr, methodErr = prov.GetFirstMethodForAction( //nolint:staticcheck,wastedassign // acceptable - retVal.GetHeirarchyIDs().GetServiceStr(), - retVal.GetHeirarchyIDs().GetResourceStr(), - methodAction, - handlerCtx.GetRuntimeContext()) - } else { - meth, methStr, methodErr = prov.GetMethodForAction( - retVal.GetHeirarchyIDs().GetServiceStr(), - retVal.GetHeirarchyIDs().GetResourceStr(), - methodAction, - parameters, - handlerCtx.GetRuntimeContext()) - if methodErr != nil { - return returnViewOnErrorIfPresent(retVal, fmt.Errorf( - "cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS IN %s: %w", //nolint:lll // long string - retVal.GetHeirarchyIDs().GetTableName(), methodErr), - viewExists) - } - } - availableServers, availableServersDoExist := svcHdl.GetServers() - if meth != nil { - availableServers, availableServersDoExist = meth.GetServers() - } - if availableServersDoExist { - for _, srv := range availableServers { - for k := range srv.Variables { - logging.GetLogger().Debugf("server parameter = '%s'\n", k) - } - } - } - method = meth - retVal.SetMethodStr(methStr) - } - if methodRequired { - retVal.SetMethod(method) - } - return retVal, nil -} - -// TODO: remove this rubbish -func returnViewOnErrorIfPresent( - input tablemetadata.HeirarchyObjects, err error, hasView bool) (tablemetadata.HeirarchyObjects, error) { - if hasView { - return input, nil - } - if err != nil { - return nil, err - } - return input, nil + return nil, fmt.Errorf("provider relations must be planned through omnisdk") } diff --git a/internal/stackql/typing/generic_typing_config.go b/internal/stackql/typing/generic_typing_config.go index 0d5802b84..6c2b226f4 100644 --- a/internal/stackql/typing/generic_typing_config.go +++ b/internal/stackql/typing/generic_typing_config.go @@ -7,8 +7,8 @@ import ( "strings" "github.com/lib/pq/oid" - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/psql-wire/pkg/sqldata" + "github.com/stackql/stackql/internal/stackql/constants" ) var ( diff --git a/internal/stackql/typing/standard_column_metadata.go b/internal/stackql/typing/standard_column_metadata.go index f196a7cbe..2ab601dcb 100644 --- a/internal/stackql/typing/standard_column_metadata.go +++ b/internal/stackql/typing/standard_column_metadata.go @@ -1,55 +1,15 @@ package typing import ( - "reflect" - "github.com/lib/pq/oid" - "github.com/stackql/any-sdk/public/formulation" "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/parserutil" -) - -var ( - _ ColumnMetadata = &standardColumnMetadata{} ) -type standardColumnMetadata struct { - coupling ORMCoupling - column formulation.ColumnDescriptor -} - -func (cd *standardColumnMetadata) GetColumnOID() oid.Oid { - return cd.getOidForSchema(cd.column.GetSchema()) -} - -func (cd *standardColumnMetadata) GetName() string { - return cd.column.GetName() -} - -func (cd *standardColumnMetadata) GetWireName() string { - return cd.column.GetWireName() -} - -func (cd *standardColumnMetadata) GetDecorated() string { - return cd.column.GetDecoratedCol() -} - -func (cd *standardColumnMetadata) GetIdentifier() string { - return cd.column.GetIdentifier() -} - -func (cd *standardColumnMetadata) GetType() string { - if cd.column.GetSchema() != nil { - return cd.column.GetSchema().GetType() - } - return parserutil.ExtractStringRepresentationOfValueColumn(cd.column.GetVal()) -} - -func (cd *standardColumnMetadata) getOidForSchema(colSchema formulation.Schema) oid.Oid { - return getOidForSchema(colSchema) +type schemaType interface { + GetType() string } -func GetOidForSchema(colSchema formulation.Schema) oid.Oid { +func GetOidForSchema(colSchema schemaType) oid.Oid { return getOidForSchema(colSchema) } @@ -78,7 +38,7 @@ func getOidForParserColType(col sqlparser.ColumnType) oid.Oid { } } -func getOidForSchema(colSchema formulation.Schema) oid.Oid { +func getOidForSchema(colSchema schemaType) oid.Oid { if colSchema == nil { return oid.T_text } @@ -93,14 +53,3 @@ func getOidForSchema(colSchema formulation.Schema) oid.Oid { return oid.T_text } } - -func (cd *standardColumnMetadata) GetRelationalType() string { - return cd.coupling.GetRelationalType() -} - -func NewColDescriptor(col formulation.ColumnDescriptor, relTypeStr string) ColumnMetadata { - return &standardColumnMetadata{ - coupling: NewORMCoupling(relTypeStr, reflect.String), - column: col, - } -} diff --git a/internal/stackql/util/annotated_tabulation.go b/internal/stackql/util/annotated_tabulation.go deleted file mode 100644 index 12e6605ec..000000000 --- a/internal/stackql/util/annotated_tabulation.go +++ /dev/null @@ -1,78 +0,0 @@ -package util //nolint:revive // fine for now - -import ( - "github.com/stackql/any-sdk/public/formulation" - "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" -) - -type AnnotatedTabulation interface { - GetAlias() string - GetHeirarchyIdentifiers() internaldto.HeirarchyIdentifiers - GetInputTableName() string - GetSQLDataSource() (sql_datasource.SQLDataSource, bool) - GetTabulation() formulation.Tabulation - SetSQLDataSource(sql_datasource.SQLDataSource) - WithParameters(parameters map[string]interface{}) AnnotatedTabulation - GetParameters() map[string]interface{} -} - -type standardAnnotatedTabulation struct { - tab formulation.Tabulation - hIDs internaldto.HeirarchyIdentifiers - inputTableName string - alias string - sqlDataSource sql_datasource.SQLDataSource - parameters map[string]interface{} -} - -func NewAnnotatedTabulation( - tab formulation.Tabulation, - hIDs internaldto.HeirarchyIdentifiers, - inputTableName string, - alias string, -) AnnotatedTabulation { - return &standardAnnotatedTabulation{ - tab: tab, - hIDs: hIDs, - inputTableName: inputTableName, - alias: alias, - parameters: make(map[string]interface{}), - } -} - -func (at *standardAnnotatedTabulation) WithParameters(parameters map[string]interface{}) AnnotatedTabulation { - at.parameters = parameters - return at -} - -func (at *standardAnnotatedTabulation) GetParameters() map[string]interface{} { - if at.parameters == nil { - return make(map[string]interface{}) - } - return at.parameters -} - -func (at *standardAnnotatedTabulation) GetTabulation() formulation.Tabulation { - return at.tab -} - -func (at *standardAnnotatedTabulation) GetAlias() string { - return at.alias -} - -func (at *standardAnnotatedTabulation) GetInputTableName() string { - return at.inputTableName -} - -func (at *standardAnnotatedTabulation) GetHeirarchyIdentifiers() internaldto.HeirarchyIdentifiers { - return at.hIDs -} - -func (at *standardAnnotatedTabulation) SetSQLDataSource(sqlDataSource sql_datasource.SQLDataSource) { - at.sqlDataSource = sqlDataSource -} - -func (at *standardAnnotatedTabulation) GetSQLDataSource() (sql_datasource.SQLDataSource, bool) { - return at.sqlDataSource, at.sqlDataSource != nil -} diff --git a/internal/stackql/util/metadata_headers.go b/internal/stackql/util/metadata_headers.go new file mode 100644 index 000000000..ebcc3701b --- /dev/null +++ b/internal/stackql/util/metadata_headers.go @@ -0,0 +1,22 @@ +package util + +func GetServicesHeader(extended bool) []string { + if extended { + return []string{"id", "name", "title", "description", "version", "preferred"} + } + return []string{"id", "name", "title"} +} + +func GetResourcesHeader(extended bool) []string { + if extended { + return []string{"name", "id", "description"} + } + return []string{"name", "id"} +} + +func GetDescribeHeader(extended bool) []string { + if extended { + return []string{"name", "type", "description"} + } + return []string{"name", "type"} +} diff --git a/internal/stackql/util/plan_utility.go b/internal/stackql/util/plan_utility.go index 5bc3caba5..774784d61 100644 --- a/internal/stackql/util/plan_utility.go +++ b/internal/stackql/util/plan_utility.go @@ -6,10 +6,9 @@ import ( "strconv" "strings" - "github.com/stackql/any-sdk/pkg/logging" - anysdk_util "github.com/stackql/any-sdk/pkg/util" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/typing" @@ -317,7 +316,7 @@ func arrangeOrderedColumnRow( case sqlparser.BoolVal: v = bool(u) } - rowVals[j] = anysdk_util.InterfaceToBytes(v, strings.ToLower(columnOrder[j]) == "error") + rowVals[j] = interfaceToBytes(v, strings.ToLower(columnOrder[j]) == "error") } return rowVals } diff --git a/internal/stackql/util/value_bytes.go b/internal/stackql/util/value_bytes.go new file mode 100644 index 000000000..bb6109a94 --- /dev/null +++ b/internal/stackql/util/value_bytes.go @@ -0,0 +1,53 @@ +package util + +import ( + "encoding/json" + "fmt" + "strconv" +) + +func interfaceToBytes(subject interface{}, isErrorCol bool) []byte { + switch sub := subject.(type) { + case bool: + if sub == true { + return []byte("true") + } + return []byte("false") + case string: + return []byte(sub) + case int: + return []byte(strconv.Itoa(sub)) + case int64: + return []byte(strconv.FormatInt(sub, 10)) + case float32: + return []byte(fmt.Sprintf("%f", sub)) + case float64: + return []byte(fmt.Sprintf("%f", sub)) + case []interface{}: + str, err := json.Marshal(subject) + if err == nil { + return str + } + return []byte(fmt.Sprintf(`{ "marshallingError": {"type": "array", "error": "%s"}}`, err.Error())) + case map[string]interface{}: + str, err := json.Marshal(subject) + if err == nil { + return str + } + return []byte(fmt.Sprintf(`{ "marshallingError": {"type": "array", "error": "%s"}}`, err.Error())) + case nil: + return []byte("null") + case interface { + Error() string + String() string + }: + if isErrorCol { + return []byte(sub.Error()) + } + return []byte(sub.String()) + default: + return []byte(fmt.Sprintf( + `{ "displayError": {"type": "%T", "error": "currently unable to represent object of type %T"}}`, + subject, subject)) + } +} diff --git a/internal/stackql/writer/writer.go b/internal/stackql/writer/writer.go index c44b4ce7f..6fdb679d6 100644 --- a/internal/stackql/writer/writer.go +++ b/internal/stackql/writer/writer.go @@ -4,7 +4,7 @@ import ( "io" "os" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/stackql/presentation" ) diff --git a/internal/stackql/writer/writer_test.go b/internal/stackql/writer/writer_test.go index 54dae0cbb..b48432d08 100644 --- a/internal/stackql/writer/writer_test.go +++ b/internal/stackql/writer/writer_test.go @@ -6,8 +6,8 @@ import ( "path/filepath" "testing" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/presentation" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/internal/test/stackqltestutil/locator.go b/internal/test/stackqltestutil/locator.go index e11e39532..8a8c8784a 100644 --- a/internal/test/stackqltestutil/locator.go +++ b/internal/test/stackqltestutil/locator.go @@ -6,9 +6,9 @@ import ( "io" "os" - "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/bundle" "github.com/stackql/stackql/internal/stackql/entryutil" + dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" "github.com/stackql/stackql/internal/stackql/util" ) @@ -39,7 +39,7 @@ func GetRuntimeCtx(providerStr string, outputFmtStr string, testName string) (*d LogLevelStr: "warn", ApplicationFilesRootPath: appRoot, AuthRaw: fmt.Sprintf(`{ "google": { "credentialsfilepath": "%s" }, "okta": { "credentialsfilepath": "%s", "type": "api_key" } }`, saKeyPath, oktaSaKeyPath), - RegistryRaw: fmt.Sprintf(`{ "url": "file://%s", "useEmbedded": false, useEmbedded, "verifyConfig": { "nopVerify": true } }`, registryRoot), + RegistryRaw: fmt.Sprintf(`{ "url": "file://%s", "localDocRoot": "%s", "useEmbedded": false, "verifyConfig": { "nopVerify": true } }`, registryRoot, registryRoot), OutputFormat: outputFmtStr, SQLBackendCfgRaw: fmt.Sprintf(`{ "dbInitFilepath": "%s", "dsn": "file:%s?mode=memory&cache=shared" }`, dbInitFilePath, testName), ExecutionConcurrencyLimit: 1, diff --git a/internal/test/stackqltestutil/setup.go b/internal/test/stackqltestutil/setup.go index 19ec07808..d5327dd83 100644 --- a/internal/test/stackqltestutil/setup.go +++ b/internal/test/stackqltestutil/setup.go @@ -4,11 +4,11 @@ package stackqltestutil import ( "fmt" "os" + "strings" "net/url" "testing" - "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/provider" "github.com/stackql/stackql/internal/stackql/util" @@ -214,18 +214,19 @@ func getNetworkInsertSuccessExpectations() map[string]testhttpapi.HTTPRequestExp nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleNetworkInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.NetworkInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.NetworkInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } func getDiskInsertSuccessExpectations(expectedRequestBody string) map[string]testhttpapi.HTTPRequestExpectations { + operationPath := strings.ReplaceAll(testobjects.GoogleComputeInsertOperationPath, "stackql-demo", "testing-project") diskInsertURL := &url.URL{ Path: testobjects.DiskInsertPath, } @@ -235,26 +236,26 @@ func getDiskInsertSuccessExpectations(expectedRequestBody string) map[string]tes "POST", diskInsertURL, testobjects.GoogleComputeHost, - testobjects.GetSimpleGoogleNetworkInsertResponse(), + strings.ReplaceAll(testobjects.GetSimpleGoogleNetworkInsertResponse(), "stackql-demo", "testing-project"), nil, ) diskInsertOpPollURL := &url.URL{ - Path: testobjects.GoogleComputeInsertOperationPath, + Path: operationPath, } networkInsertOpPollExpectation := testhttpapi.NewHTTPRequestExpectations( nil, nil, "GET", diskInsertOpPollURL, - testobjects.GoogleApisHost, - testobjects.GetSimplePollOperationGoogleNetworkInsertResponse(), + testobjects.GoogleComputeHost, + strings.ReplaceAll(testobjects.GetSimplePollOperationGoogleNetworkInsertResponse(), "stackql-demo", "testing-project"), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.DiskInsertPath: diskInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.DiskInsertPath: diskInsertExpectation, + testobjects.GoogleComputeHost + operationPath: networkInsertOpPollExpectation, } } @@ -299,14 +300,14 @@ func getSubnetworkInsertSuccessExpectations() map[string]testhttpapi.HTTPRequest nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleSubnetworkInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.SubnetworkInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.SubnetworkInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -332,14 +333,14 @@ func getIPInsertSuccessExpectations() map[string]testhttpapi.HTTPRequestExpectat nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleIPInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.IPInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.IPInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -365,14 +366,14 @@ func getInternalFirewallInsertSuccessExpectations() map[string]testhttpapi.HTTPR nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleFirewallInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.FirewallInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.FirewallInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -398,14 +399,14 @@ func getExternalFirewallInsertSuccessExpectations() map[string]testhttpapi.HTTPR nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleFirewallInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.FirewallInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.FirewallInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -431,14 +432,14 @@ func getComputeInstanceInsertSuccessExpectations(name string, secondaryTag strin nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleComputeInstanceInsertResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + testobjects.ComputeInstanceInsertPath: networkInsertExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + testobjects.ComputeInstanceInsertPath: networkInsertExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -465,14 +466,14 @@ func getNetworkDeleteSuccessExpectations() map[string]testhttpapi.HTTPRequestExp nil, "GET", networkInsertOpPollURL, - testobjects.GoogleApisHost, + testobjects.GoogleComputeHost, testobjects.GetSimplePollOperationGoogleNetworkDeleteResponse(), nil, ) return map[string]testhttpapi.HTTPRequestExpectations{ - testobjects.GoogleComputeHost + path: networkDeleteExpectation, - testobjects.GoogleApisHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, + testobjects.GoogleComputeHost + path: networkDeleteExpectation, + testobjects.GoogleComputeHost + testobjects.GoogleComputeInsertOperationPath: networkInsertOpPollExpectation, } } @@ -483,7 +484,6 @@ func SetupSimpleInsertGoogleComputeNetworks(t *testing.T) { } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } func SetupDependentInsertGoogleComputeDisks(t *testing.T) { @@ -492,14 +492,13 @@ func SetupDependentInsertGoogleComputeDisks(t *testing.T) { expectations.Put(k, v) } for k, v := range getDiskInsertSuccessExpectations(testobjects.CreateGoogleComputeDiskRequestPayload01) { - expectations.Put(k, v) + expectations.PutUnordered(t, k, v) } for k, v := range getDiskInsertSuccessExpectations(testobjects.CreateGoogleComputeDiskRequestPayload02) { - expectations.Put(k, v) + expectations.PutUnordered(t, k, v) } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } func SetupExecGoogleOrganizationsGetIamPolicy(t *testing.T) { @@ -513,18 +512,17 @@ func SetupExecGoogleOrganizationsGetIamPolicy(t *testing.T) { func SetupDependentInsertGoogleComputeDisksReversed(t *testing.T) { expectations := testhttpapi.NewExpectationStore(5) //nolint:mnd // TODO: tech debt sweep mnd hacks - for k, v := range getCloudResourceManagerProjectSelectExpectations(t) { + for k, v := range getDisksSelectExpectations(t) { expectations.Put(k, v) } for k, v := range getDiskInsertSuccessExpectations(testobjects.CreateGoogleComputeDiskRequestPayload03) { - expectations.Put(k, v) + expectations.PutUnordered(t, k, v) } for k, v := range getDiskInsertSuccessExpectations(testobjects.CreateGoogleComputeDiskRequestPayload04) { - expectations.Put(k, v) + expectations.PutUnordered(t, k, v) } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } func SetupDependentInsertGoogleBQDatasets(t *testing.T) { @@ -556,7 +554,6 @@ func SetupDependentInsertGoogleBQDatasets(t *testing.T) { } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } func SetupSimpleDeleteGoogleComputeNetworks(t *testing.T) { @@ -566,7 +563,6 @@ func SetupSimpleDeleteGoogleComputeNetworks(t *testing.T) { } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } func SetupK8sTheHardWayE2eSuccess(t *testing.T) { @@ -603,7 +599,6 @@ func SetupK8sTheHardWayE2eSuccess(t *testing.T) { } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } //nolint:mnd // TODO: tech debt sweep mnd hacks @@ -639,5 +634,4 @@ func SetupInsertDependentComputeDiskSuccess(t *testing.T) { } testhttpapi.StartServer(t, expectations) provider.DummyAuth = true - execution.MonitorPollIntervalSeconds = 2 } diff --git a/internal/test/testhttpapi/httptestutils.go b/internal/test/testhttpapi/httptestutils.go index 2a9be3ecc..b7a9c227f 100644 --- a/internal/test/testhttpapi/httptestutils.go +++ b/internal/test/testhttpapi/httptestutils.go @@ -3,6 +3,7 @@ package testhttpapi import ( "bytes" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -15,7 +16,7 @@ import ( "sync" "testing" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" "github.com/stackql/stackql/internal/test/testobjects" "github.com/stackql/stackql/internal/test/testutil" ) @@ -115,6 +116,7 @@ func newSimpleTransportHandler(ex ExpectationStore) func(*http.Request) (*http.R if req.URL.RawQuery != "" { keyStr = keyStr + "?" + req.URL.RawQuery } + keyStr = bodyExpectationKey(req, ex, keyStr) expectations, ok := ex.Get(keyStr) err := compareHTTPRequestToExpected(req, &expectations) if err != nil { @@ -151,11 +153,16 @@ func (srt SimulatedRoundTripper) RoundTrip(req *http.Request) (*http.Response, e if req.URL.RawQuery != "" { reqKey = reqKey + "?" + req.URL.RawQuery } + reqKey = bodyExpectationKey(req, srt.Expectations, reqKey) ok := srt.Expectations.HasKey(reqKey) if !ok && srt.Strict { srt.T.Fatalf("FAIL: no expectations found for key '%s' in strict mode, existing keys = %s", reqKey, strings.Join(srt.Expectations.Keys(), ", ")) } - return srt.RoundTripper(req) + response, err := srt.RoundTripper(req) + if err != nil { + srt.T.Logf("%s %s: %v", req.Method, reqKey, err) + } + return response, err } func NewURL(scheme, host, path string) *url.URL { @@ -472,3 +479,38 @@ func StartServer(t testing.TB, expectations ExpectationStore) { var roundTripper http.RoundTripper = NewSimulatedRoundTripper(t, expectations, transport, true) SetupHTTPCallHeavyweight(t, expectations, DefaultHandler, roundTripper) } + +// PutUnordered retains exact payload and count checks for independent mutation rows. +func (ex ExpectationStore) PutUnordered(t testing.TB, key string, value HTTPRequestExpectations) { + t.Helper() + if value.Body == nil { + ex.Put(key, value) + return + } + body, err := io.ReadAll(value.Body) + if err != nil { + t.Fatal(err) + } + value.Body = io.NopCloser(bytes.NewReader(body)) + ex[key+"#body"] = &ExpectationList{} + ex.Put(key+bodyFingerprint(body), value) +} + +func bodyFingerprint(body []byte) string { + var value any + if json.Unmarshal(body, &value) == nil { + body, _ = json.Marshal(value) + } + return fmt.Sprintf("#%x", sha256.Sum256(body)) +} +func bodyExpectationKey(req *http.Request, ex ExpectationStore, key string) string { + if _, matched := ex[key+"#body"]; !matched || req.Body == nil { + return key + } + body, err := io.ReadAll(req.Body) + req.Body = io.NopCloser(bytes.NewReader(body)) + if err != nil || len(body) == 0 { + return key + } + return key + bodyFingerprint(body) +} diff --git a/pkg/astformat/ast_format_postgres.go b/pkg/astformat/ast_format_postgres.go index ec07d3d3a..995d5cabe 100644 --- a/pkg/astformat/ast_format_postgres.go +++ b/pkg/astformat/ast_format_postgres.go @@ -4,8 +4,9 @@ import ( "fmt" "strings" - "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/astfuncrewrite" + "github.com/stackql/stackql/internal/stackql/constants" ) //nolint:revive // Explicit type declaration removes any ambiguity @@ -27,8 +28,9 @@ func PostgresSelectExprsFormatter(buf *sqlparser.TrackedBuffer, node sqlparser.S formatColIdent(node, buf) return case *sqlparser.FuncExpr: + node = postgresJSONExtract(node) if strings.ToLower(node.Name.GetRawVal()) == constants.SQLFuncJSONExtractPostgres && len(node.Exprs) > 1 { - sb := sqlparser.NewTrackedBuffer(PostgresSelectExprsFormatter) + sb := buf sb.AstPrintf( node, "%s(%v%s, %v", @@ -40,11 +42,10 @@ func PostgresSelectExprsFormatter(buf *sqlparser.TrackedBuffer, node sqlparser.S sb.AstPrintf(node, ", %v", val) } sb.AstPrintf(node, ")") - buf.WriteString(sb.String()) return } if strings.ToLower(node.Name.GetRawVal()) == sqlparser.JsonArrayElementsTextStr && len(node.Exprs) >= 1 { - sb := sqlparser.NewTrackedBuffer(PostgresSelectExprsFormatter) + sb := buf sb.AstPrintf( node, "%s(%v%s", @@ -58,15 +59,13 @@ func PostgresSelectExprsFormatter(buf *sqlparser.TrackedBuffer, node sqlparser.S } } sb.AstPrintf(node, ")") - buf.WriteString(sb.String()) return } node.Format(buf) return case *sqlparser.GroupConcatExpr: - sb := sqlparser.NewTrackedBuffer(PostgresSelectExprsFormatter) + sb := buf sb.AstPrintf(node, "%s(%v, %s)", constants.SQLFuncGroupConcatPostgres, node.Exprs[0], `','`) - buf.WriteString(sb.String()) return default: @@ -117,3 +116,15 @@ func formattedString(node sqlparser.SQLNode, formatter sqlparser.NodeFormatter) buf.Myprintf("%v", node) return buf.String() } + +func postgresJSONExtract(node *sqlparser.FuncExpr) *sqlparser.FuncExpr { + if !strings.EqualFold(node.Name.GetRawVal(), constants.SQLFuncJSONExtractConformed) { + return node + } + cloned := *node + rewritten, err := astfuncrewrite.GetPostgresASTFuncRewriter().RewriteFunc(&cloned) + if err != nil { + return node + } + return rewritten +} diff --git a/stackql/main_test.go b/stackql/main_test.go index e5f6ec39b..c6db91af0 100644 --- a/stackql/main_test.go +++ b/stackql/main_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/stackql/internal/stackql/logging" ) func TestMain(m *testing.M) { diff --git a/test/assets/expected/aws/ce/ce-cast-real-postgres.txt b/test/assets/expected/aws/ce/ce-cast-real-postgres.txt index 018e2a202..bf6be6483 100644 --- a/test/assets/expected/aws/ce/ce-cast-real-postgres.txt +++ b/test/assets/expected/aws/ce/ce-cast-real-postgres.txt @@ -1,153 +1,153 @@ -|------------|------------|--------------------------------|------------------------|------| -| beginning | ending | keyz | amount | unit | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS CloudFormation"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["AWS Glue"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["AWS Glue"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["AWS Glue"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS Glue"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS Lambda"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["AWS Secrets Manager"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["AWS Secrets Manager"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["AWS Secrets Manager"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS Secrets Manager"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Simple Notification | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Simple Notification | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Simple Notification | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Simple Notification | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Simple Queue | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Simple Queue | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Simple Queue | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Simple Queue | 0 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["AmazonCloudWatch"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["AmazonCloudWatch"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["AmazonCloudWatch"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AmazonCloudWatch"] | 0 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS CloudShell"] | 2.1520000359487312e-07 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Location Service"] | 0.0014550000196322799 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Location Service"] | 0.0015539999585598707 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Location Service"] | 0.0015839999541640282 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Location Service"] | 0.0016530000139027834 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon DynamoDB"] | 0.0051069543696939945 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Simple Storage | 0.04730991646647453 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon EC2 Container | 0.06115703284740448 | USD | -| | | Registry (ECR)"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon EC2 Container | 0.06326588988304138 | USD | -| | | Registry (ECR)"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon EC2 Container | 0.06326588988304138 | USD | -| | | Registry (ECR)"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon EC2 Container | 0.06326589733362198 | USD | -| | | Registry (ECR)"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon DynamoDB"] | 0.06669150292873383 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon DynamoDB"] | 0.0787472054362297 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon DynamoDB"] | 0.12785232067108154 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Simple Storage | 0.35454288125038147 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Simple Storage | 0.3908839821815491 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Simple Storage | 0.6006374359130859 | USD | -| | | Service"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["AWS Key Management Service"] | 1 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS Config"] | 1.6790000200271606 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["AWS Config"] | 1.7589999437332153 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["AWS Config"] | 1.805999994277954 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["EC2 - Other"] | 1.832593321800232 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["AWS Config"] | 1.9819999933242798 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["EC2 - Other"] | 2.3042662143707275 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["EC2 - Other"] | 2.304311513900757 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Tax"] | 3.4200000762939453 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["AWS Key Management Service"] | 4.1631550788879395 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Tax"] | 4.579999923706055 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Tax"] | 4.940000057220459 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["AWS Key Management Service"] | 6.3830108642578125 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS Key Management Service"] | 6.405858516693115 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Virtual Private | 8.857989311218262 | USD | -| | | Cloud"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Virtual Private | 10.800800323486328 | USD | -| | | Cloud"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Virtual Private | 11.15999984741211 | USD | -| | | Cloud"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Tax"] | 14.039999961853027 | USD | -|------------|------------|--------------------------------|------------------------|------| -| 2024-08-01 | 2024-09-01 | ["Amazon Elastic Compute Cloud | 20.445402145385742 | USD | -| | | - Compute"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-09-01 | 2024-10-01 | ["Amazon Elastic Compute Cloud | 26.282133102416992 | USD | -| | | - Compute"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-10-01 | 2024-11-01 | ["Amazon Elastic Compute Cloud | 27.126800537109375 | USD | -| | | - Compute"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Elastic Compute Cloud | 27.15462303161621 | USD | -| | | - Compute"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["Amazon Virtual Private | 47.563499450683594 | USD | -| | | Cloud"] | | | -|------------|------------|--------------------------------|------------------------|------| -| 2024-11-01 | 2024-11-30 | ["EC2 - Other"] | 50.93074035644531 | USD | -|------------|------------|--------------------------------|------------------------|------| \ No newline at end of file +|------------|------------|--------------------------------|---------------------------|------| +| beginning | ending | keyz | amount | unit | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS CloudFormation"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["AWS Glue"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["AWS Glue"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["AWS Glue"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS Glue"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS Lambda"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["AWS Secrets Manager"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["AWS Secrets Manager"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["AWS Secrets Manager"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS Secrets Manager"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Simple Notification | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Simple Notification | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Simple Notification | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Simple Notification | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Simple Queue | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Simple Queue | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Simple Queue | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Simple Queue | 0 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["AmazonCloudWatch"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["AmazonCloudWatch"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["AmazonCloudWatch"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AmazonCloudWatch"] | 0 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS CloudShell"] | 0.00000021520000359487312 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Location Service"] | 0.0014550000196322799 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Location Service"] | 0.0015539999585598707 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Location Service"] | 0.0015839999541640282 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Location Service"] | 0.0016530000139027834 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon DynamoDB"] | 0.0051069543696939945 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Simple Storage | 0.04730991646647453 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon EC2 Container | 0.06115703284740448 | USD | +| | | Registry (ECR)"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon EC2 Container | 0.06326588988304138 | USD | +| | | Registry (ECR)"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon EC2 Container | 0.06326588988304138 | USD | +| | | Registry (ECR)"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon EC2 Container | 0.06326589733362198 | USD | +| | | Registry (ECR)"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon DynamoDB"] | 0.06669150292873383 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon DynamoDB"] | 0.0787472054362297 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon DynamoDB"] | 0.12785232067108154 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Simple Storage | 0.35454288125038147 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Simple Storage | 0.3908839821815491 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Simple Storage | 0.6006374359130859 | USD | +| | | Service"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["AWS Key Management Service"] | 1 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS Config"] | 1.6790000200271606 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["AWS Config"] | 1.7589999437332153 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["AWS Config"] | 1.805999994277954 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["EC2 - Other"] | 1.832593321800232 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["AWS Config"] | 1.9819999933242798 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["EC2 - Other"] | 2.3042662143707275 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["EC2 - Other"] | 2.304311513900757 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Tax"] | 3.4200000762939453 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["AWS Key Management Service"] | 4.1631550788879395 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Tax"] | 4.579999923706055 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Tax"] | 4.940000057220459 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["AWS Key Management Service"] | 6.3830108642578125 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["AWS Key Management Service"] | 6.405858516693115 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Virtual Private | 8.857989311218262 | USD | +| | | Cloud"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Virtual Private | 10.800800323486328 | USD | +| | | Cloud"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Virtual Private | 11.15999984741211 | USD | +| | | Cloud"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Tax"] | 14.039999961853027 | USD | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-08-01 | 2024-09-01 | ["Amazon Elastic Compute Cloud | 20.445402145385742 | USD | +| | | - Compute"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-09-01 | 2024-10-01 | ["Amazon Elastic Compute Cloud | 26.282133102416992 | USD | +| | | - Compute"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-10-01 | 2024-11-01 | ["Amazon Elastic Compute Cloud | 27.126800537109375 | USD | +| | | - Compute"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Elastic Compute Cloud | 27.15462303161621 | USD | +| | | - Compute"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["Amazon Virtual Private | 47.563499450683594 | USD | +| | | Cloud"] | | | +|------------|------------|--------------------------------|---------------------------|------| +| 2024-11-01 | 2024-11-30 | ["EC2 - Other"] | 50.93074035644531 | USD | +|------------|------------|--------------------------------|---------------------------|------| \ No newline at end of file diff --git a/test/assets/expected/aws/ce/ce-cast-real.txt b/test/assets/expected/aws/ce/ce-cast-real.txt index 58e7a661b..9795862c3 100644 --- a/test/assets/expected/aws/ce/ce-cast-real.txt +++ b/test/assets/expected/aws/ce/ce-cast-real.txt @@ -53,7 +53,7 @@ |------------|------------|--------------------------------|---------------|------| | 2024-11-01 | 2024-11-30 | ["AmazonCloudWatch"] | 0 | USD | |------------|------------|--------------------------------|---------------|------| -| 2024-11-01 | 2024-11-30 | ["AWS CloudShell"] | 2.152e-07 | USD | +| 2024-11-01 | 2024-11-30 | ["AWS CloudShell"] | 0.0000002152 | USD | |------------|------------|--------------------------------|---------------|------| | 2024-11-01 | 2024-11-30 | ["Amazon Location Service"] | 0.001455 | USD | |------------|------------|--------------------------------|---------------|------| diff --git a/test/assets/expected/github/joins/specialcase-firstlook-repositories-collaborators.json b/test/assets/expected/github/joins/specialcase-firstlook-repositories-collaborators.json index 930906711..9b150e941 100644 --- a/test/assets/expected/github/joins/specialcase-firstlook-repositories-collaborators.json +++ b/test/assets/expected/github/joins/specialcase-firstlook-repositories-collaborators.json @@ -1,14 +1,14 @@ [ - { "id": "100000001", "name": "docusaurus-plugin-hubspot", "login": "somejimbo", "type": "User", "role_name": "admin" }, - { "id": "100000001", "name": "docusaurus-plugin-hubspot", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, - { "id": "100000001", "name": "docusaurus-plugin-hubspot", "login": "johnsmith", "type": "User", "role_name": "read" }, - { "id": "100000001", "name": "docusaurus-plugin-hubspot", "login": "joeblow", "type": "User", "role_name": "admin" }, - { "id": "100000002", "name": "dummyapp.io", "login": "somejimbo", "type": "User", "role_name": "admin" }, - { "id": "100000002", "name": "dummyapp.io", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, - { "id": "100000002", "name": "dummyapp.io", "login": "johnsmith", "type": "User", "role_name": "read" }, - { "id": "100000002", "name": "dummyapp.io", "login": "joeblow", "type": "User", "role_name": "admin" }, - { "id": "100000003", "name": "gatsby-plugin-smartlook", "login": "somejimbo", "type": "User", "role_name": "admin" }, - { "id": "100000003", "name": "gatsby-plugin-smartlook", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, - { "id": "100000003", "name": "gatsby-plugin-smartlook", "login": "johnsmith", "type": "User", "role_name": "read" }, - { "id": "100000003", "name": "gatsby-plugin-smartlook", "login": "joeblow", "type": "User", "role_name": "admin" } + { "id": 100000001, "name": "docusaurus-plugin-hubspot", "login": "somejimbo", "type": "User", "role_name": "admin" }, + { "id": 100000001, "name": "docusaurus-plugin-hubspot", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, + { "id": 100000001, "name": "docusaurus-plugin-hubspot", "login": "johnsmith", "type": "User", "role_name": "read" }, + { "id": 100000001, "name": "docusaurus-plugin-hubspot", "login": "joeblow", "type": "User", "role_name": "admin" }, + { "id": 100000002, "name": "dummyapp.io", "login": "somejimbo", "type": "User", "role_name": "admin" }, + { "id": 100000002, "name": "dummyapp.io", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, + { "id": 100000002, "name": "dummyapp.io", "login": "johnsmith", "type": "User", "role_name": "read" }, + { "id": 100000002, "name": "dummyapp.io", "login": "joeblow", "type": "User", "role_name": "admin" }, + { "id": 100000003, "name": "gatsby-plugin-smartlook", "login": "somejimbo", "type": "User", "role_name": "admin" }, + { "id": 100000003, "name": "gatsby-plugin-smartlook", "login": "marcusaurelius", "type": "User", "role_name": "admin" }, + { "id": 100000003, "name": "gatsby-plugin-smartlook", "login": "johnsmith", "type": "User", "role_name": "read" }, + { "id": 100000003, "name": "gatsby-plugin-smartlook", "login": "joeblow", "type": "User", "role_name": "admin" } ] diff --git a/test/assets/expected/k8s-the-hard-way/k8s-the-hard-way-e2e/success.txt b/test/assets/expected/k8s-the-hard-way/k8s-the-hard-way-e2e/success.txt index 1ee30d2c2..195e7ef84 100644 --- a/test/assets/expected/k8s-the-hard-way/k8s-the-hard-way-e2e/success.txt +++ b/test/assets/expected/k8s-the-hard-way/k8s-the-hard-way-e2e/success.txt @@ -1,22 +1,22 @@ -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete diff --git a/test/assets/expected/show/show-insert-google-compute-instances-iam-error.txt b/test/assets/expected/show/show-insert-google-compute-instances-iam-error.txt index f59c1a08d..a19e96602 100644 --- a/test/assets/expected/show/show-insert-google-compute-instances-iam-error.txt +++ b/test/assets/expected/show/show-insert-google-compute-instances-iam-error.txt @@ -1 +1 @@ -cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS IN google.compute.instances_iam_policies: cannot resolve Method \ No newline at end of file +stackqldoc: resource "instances_iam_policies" declares no insert verb \ No newline at end of file diff --git a/test/assets/expected/show/show-insert-google-iam-service-accounts.txt b/test/assets/expected/show/show-insert-google-iam-service-accounts.txt index 18d6c7b89..54e1c59ed 100644 --- a/test/assets/expected/show/show-insert-google-iam-service-accounts.txt +++ b/test/assets/expected/show/show-insert-google-iam-service-accounts.txt @@ -1,38 +1,26 @@ -|---------------------------------------------| -| insert_statement | -|---------------------------------------------| -| <<>, data__accountId: | -| << data__accountId >>, | -| data__serviceAccount: { description: | -| << description >>, disabled: << | -| disabled >>, displayName: << displayName | -| >>, email: << email >>, etag: | -| << etag >>, name: << name >>, | -| oauth2ClientId: << oauth2ClientId >>, | -| projectId: << projectId >>, uniqueId: | -| << uniqueId >> } } >>> INSERT INTO | -| google.iam.service_accounts( projectsId, | -| data__accountId, data__serviceAccount | -| ) SELECT '{{ .values.projectsId | -| }}', '{{ .values.data__accountId | -| }}', '{ "description": "{{ | -| .values.data__serviceAccount.description | -| }}", "disabled": {{ | -| .values.data__serviceAccount.disabled | -| }}, "displayName": "{{ | -| .values.data__serviceAccount.displayName | -| }}", "email": "{{ | -| .values.data__serviceAccount.email | -| }}", "etag": "{{ | -| .values.data__serviceAccount.etag | -| }}", "name": "{{ | -| .values.data__serviceAccount.name | -| }}", "oauth2ClientId": "{{ | -| .values.data__serviceAccount.oauth2ClientId | -| }}", "projectId": "{{ | -| .values.data__serviceAccount.projectId | -| }}", "uniqueId": "{{ | -| .values.data__serviceAccount.uniqueId }}" | -| }' ; | -|---------------------------------------------| \ No newline at end of file +|------------------------------------------| +| insert_statement | +|------------------------------------------| +| <<>, data__accountId: | +| << data__accountId >>, | +| data__serviceAccount: { | +| description: << description >>, | +| displayName: << displayName >>, | +| etag: << etag >>, name: << | +| name >> } } >>> INSERT INTO | +| google.iam.service_accounts( | +| projectsId, data__accountId, | +| data__serviceAccount ) SELECT | +| '{{ .values.projectsId }}', | +| '{{ .values.data__accountId | +| }}', '{ "description": "{{ | +| .values.data__serviceAccount.description | +| }}", "displayName": "{{ | +| .values.data__serviceAccount.displayName | +| }}", "etag": "{{ | +| .values.data__serviceAccount.etag | +| }}", "name": "{{ | +| .values.data__serviceAccount.name }}" }' | +| ; | +|------------------------------------------| diff --git a/test/assets/expected/show/show-resources-filtered.csv b/test/assets/expected/show/show-resources-filtered.csv index d0bf4a94c..57cfa5c81 100644 --- a/test/assets/expected/show/show-resources-filtered.csv +++ b/test/assets/expected/show/show-resources-filtered.csv @@ -1,2 +1,4 @@ instances,google.compute.instances instances_iam_policies,google.compute.instances_iam_policies +regionInstances,google.compute.regionInstances +targetInstances,google.compute.targetInstances diff --git a/test/assets/expected/simple-delete/compute-network/delete-compute-network.txt b/test/assets/expected/simple-delete/compute-network/delete-compute-network.txt index b37642311..6f03dbd2f 100644 --- a/test/assets/expected/simple-delete/compute-network/delete-compute-network.txt +++ b/test/assets/expected/simple-delete/compute-network/delete-compute-network.txt @@ -1,2 +1,2 @@ -compute#operation: delete in progress, 2 seconds elapsed +compute#operation: delete in progress, 10 seconds elapsed compute#operation: delete complete diff --git a/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk-parallel.txt b/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk-parallel.txt new file mode 100644 index 000000000..82da66ac2 --- /dev/null +++ b/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk-parallel.txt @@ -0,0 +1,4 @@ +compute#operation: insert in progress, 10 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed +compute#operation: insert complete +compute#operation: insert complete diff --git a/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk.txt b/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk.txt index 0adcc136d..86aa87800 100644 --- a/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk.txt +++ b/test/assets/expected/simple-insert/compute-disk/insert-dependent-compute-disk.txt @@ -1,4 +1,4 @@ -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete diff --git a/test/assets/expected/simple-insert/compute-network/insert-compute-network.txt b/test/assets/expected/simple-insert/compute-network/insert-compute-network.txt index 762d49bf1..213b7cf9e 100644 --- a/test/assets/expected/simple-insert/compute-network/insert-compute-network.txt +++ b/test/assets/expected/simple-insert/compute-network/insert-compute-network.txt @@ -1,2 +1,2 @@ -compute#operation: insert in progress, 2 seconds elapsed +compute#operation: insert in progress, 10 seconds elapsed compute#operation: insert complete diff --git a/test/assets/expected/simple-templating/insert-bigquery-datasets-extended.iql b/test/assets/expected/simple-templating/insert-bigquery-datasets-extended.iql index 9433fcd51..fe85306d3 100644 --- a/test/assets/expected/simple-templating/insert-bigquery-datasets-extended.iql +++ b/test/assets/expected/simple-templating/insert-bigquery-datasets-extended.iql @@ -31,9 +31,7 @@ data__description: << data__description >>, data__friendlyName: << data__friendlyName >>, data__isCaseInsensitive: << data__isCaseInsensitive >>, - data__labels: { - << data__labels[0].key >>: << data__labels[0].val >> - }, + data__labels: << data__labels >>, data__location: << data__location >> } >>> @@ -66,7 +64,7 @@ SELECT '{{ .values.projectId }}', '{{ .values.quotaUser }}', '{{ .values.userIp }}', - '[ { "dataset": "{{ .values.data__access[0].dataset }}", "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": "{{ .values.data__access[0].routine }}", "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": "{{ .values.data__access[0].view }}" } ]', + '[ { "dataset": {{ .values.data__access[0].dataset }}, "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": {{ .values.data__access[0].routine }}, "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": {{ .values.data__access[0].view }} } ]', '{ "datasetId": "{{ .values.data__datasetReference.datasetId }}", "projectId": "{{ .values.data__datasetReference.projectId }}" }', '{ "kmsKeyName": "{{ .values.data__defaultEncryptionConfiguration.kmsKeyName }}" }', '{{ .values.data__defaultPartitionExpirationMs }}', @@ -74,6 +72,6 @@ SELECT '{{ .values.data__description }}', '{{ .values.data__friendlyName }}', {{ .values.data__isCaseInsensitive }}, - '{ "{{ .values.data__labels[0].key }}": "{{ .values.data__labels[0].val }}" }', + {{ .values.data__labels }}, '{{ .values.data__location }}' -; \ No newline at end of file +; diff --git a/test/assets/expected/simple-templating/insert-bigquery-datasets.iql b/test/assets/expected/simple-templating/insert-bigquery-datasets.iql index e9802e64a..075ddd7e7 100644 --- a/test/assets/expected/simple-templating/insert-bigquery-datasets.iql +++ b/test/assets/expected/simple-templating/insert-bigquery-datasets.iql @@ -24,9 +24,7 @@ data__description: << data__description >>, data__friendlyName: << data__friendlyName >>, data__isCaseInsensitive: << data__isCaseInsensitive >>, - data__labels: { - << data__labels[0].key >>: << data__labels[0].val >> - }, + data__labels: << data__labels >>, data__location: << data__location >> } >>> @@ -45,7 +43,7 @@ INSERT INTO google.bigquery.datasets( ) SELECT '{{ .values.projectId }}', - '[ { "dataset": "{{ .values.data__access[0].dataset }}", "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": "{{ .values.data__access[0].routine }}", "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": "{{ .values.data__access[0].view }}" } ]', + '[ { "dataset": {{ .values.data__access[0].dataset }}, "domain": "{{ .values.data__access[0].domain }}", "groupByEmail": "{{ .values.data__access[0].groupByEmail }}", "iamMember": "{{ .values.data__access[0].iamMember }}", "role": "{{ .values.data__access[0].role }}", "routine": {{ .values.data__access[0].routine }}, "specialGroup": "{{ .values.data__access[0].specialGroup }}", "userByEmail": "{{ .values.data__access[0].userByEmail }}", "view": {{ .values.data__access[0].view }} } ]', '{ "datasetId": "{{ .values.data__datasetReference.datasetId }}", "projectId": "{{ .values.data__datasetReference.projectId }}" }', '{ "kmsKeyName": "{{ .values.data__defaultEncryptionConfiguration.kmsKeyName }}" }', '{{ .values.data__defaultPartitionExpirationMs }}', @@ -53,6 +51,6 @@ SELECT '{{ .values.data__description }}', '{{ .values.data__friendlyName }}', {{ .values.data__isCaseInsensitive }}, - '{ "{{ .values.data__labels[0].key }}": "{{ .values.data__labels[0].val }}" }', + {{ .values.data__labels }}, '{{ .values.data__location }}' -; \ No newline at end of file +; diff --git a/test/assets/expected/simple-templating/insert-container-clusters.iql b/test/assets/expected/simple-templating/insert-container-clusters.iql index e2c3114ae..174c2abf9 100644 --- a/test/assets/expected/simple-templating/insert-container-clusters.iql +++ b/test/assets/expected/simple-templating/insert-container-clusters.iql @@ -22,7 +22,6 @@ enableTpu: << enableTpu >>, endpoint: << endpoint >>, expireTime: << expireTime >>, - id: << id >>, initialClusterVersion: << initialClusterVersion >>, initialNodeCount: << initialNodeCount >>, instanceGroupUrls: [ << instanceGroupUrls[0] >> ], @@ -50,9 +49,7 @@ notificationConfig: << notificationConfig >>, privateClusterConfig: << privateClusterConfig >>, releaseChannel: << releaseChannel >>, - resourceLabels: { - << resourceLabels[0].key >>: << resourceLabels[0].val >> - }, + resourceLabels: << resourceLabels >>, resourceUsageExportConfig: << resourceUsageExportConfig >>, selfLink: << selfLink >>, servicesIpv4Cidr: << servicesIpv4Cidr >>, @@ -81,8 +78,8 @@ INSERT INTO google.container.projects.zones.clusters( SELECT '{{ .values.projectId }}', '{{ .values.zone }}', - '{ "addonsConfig": "{{ .values.data__cluster.addonsConfig }}", "authenticatorGroupsConfig": "{{ .values.data__cluster.authenticatorGroupsConfig }}", "autopilot": "{{ .values.data__cluster.autopilot }}", "autoscaling": "{{ .values.data__cluster.autoscaling }}", "binaryAuthorization": "{{ .values.data__cluster.binaryAuthorization }}", "clusterIpv4Cidr": "{{ .values.data__cluster.clusterIpv4Cidr }}", "conditions": [ "{{ .values.data__cluster.conditions[0] }}" ], "confidentialNodes": "{{ .values.data__cluster.confidentialNodes }}", "createTime": "{{ .values.data__cluster.createTime }}", "currentMasterVersion": "{{ .values.data__cluster.currentMasterVersion }}", "currentNodeCount": {{ .values.data__cluster.currentNodeCount }}, "currentNodeVersion": "{{ .values.data__cluster.currentNodeVersion }}", "databaseEncryption": "{{ .values.data__cluster.databaseEncryption }}", "defaultMaxPodsConstraint": "{{ .values.data__cluster.defaultMaxPodsConstraint }}", "description": "{{ .values.data__cluster.description }}", "enableKubernetesAlpha": {{ .values.data__cluster.enableKubernetesAlpha }}, "enableTpu": {{ .values.data__cluster.enableTpu }}, "endpoint": "{{ .values.data__cluster.endpoint }}", "expireTime": "{{ .values.data__cluster.expireTime }}", "id": "{{ .values.data__cluster.id }}", "initialClusterVersion": "{{ .values.data__cluster.initialClusterVersion }}", "initialNodeCount": {{ .values.data__cluster.initialNodeCount }}, "instanceGroupUrls": [ "{{ .values.data__cluster.instanceGroupUrls[0] }}" ], "ipAllocationPolicy": "{{ .values.data__cluster.ipAllocationPolicy }}", "labelFingerprint": "{{ .values.data__cluster.labelFingerprint }}", "legacyAbac": "{{ .values.data__cluster.legacyAbac }}", "location": "{{ .values.data__cluster.location }}", "locations": [ "{{ .values.data__cluster.locations[0] }}" ], "loggingConfig": "{{ .values.data__cluster.loggingConfig }}", "loggingService": "{{ .values.data__cluster.loggingService }}", "maintenancePolicy": "{{ .values.data__cluster.maintenancePolicy }}", "masterAuth": "{{ .values.data__cluster.masterAuth }}", "masterAuthorizedNetworksConfig": "{{ .values.data__cluster.masterAuthorizedNetworksConfig }}", "meshCertificates": "{{ .values.data__cluster.meshCertificates }}", "monitoringConfig": "{{ .values.data__cluster.monitoringConfig }}", "monitoringService": "{{ .values.data__cluster.monitoringService }}", "name": "{{ .values.data__cluster.name }}", "network": "{{ .values.data__cluster.network }}", "networkConfig": "{{ .values.data__cluster.networkConfig }}", "networkPolicy": "{{ .values.data__cluster.networkPolicy }}", "nodeConfig": "{{ .values.data__cluster.nodeConfig }}", "nodeIpv4CidrSize": {{ .values.data__cluster.nodeIpv4CidrSize }}, "nodePoolDefaults": "{{ .values.data__cluster.nodePoolDefaults }}", "nodePools": [ "{{ .values.data__cluster.nodePools[0] }}" ], "notificationConfig": "{{ .values.data__cluster.notificationConfig }}", "privateClusterConfig": "{{ .values.data__cluster.privateClusterConfig }}", "releaseChannel": "{{ .values.data__cluster.releaseChannel }}", "resourceLabels": { "{{ .values.data__cluster.resourceLabels[0].key }}": "{{ .values.data__cluster.resourceLabels[0].val }}" }, "resourceUsageExportConfig": "{{ .values.data__cluster.resourceUsageExportConfig }}", "selfLink": "{{ .values.data__cluster.selfLink }}", "servicesIpv4Cidr": "{{ .values.data__cluster.servicesIpv4Cidr }}", "shieldedNodes": "{{ .values.data__cluster.shieldedNodes }}", "status": "{{ .values.data__cluster.status }}", "statusMessage": "{{ .values.data__cluster.statusMessage }}", "subnetwork": "{{ .values.data__cluster.subnetwork }}", "tpuIpv4CidrBlock": "{{ .values.data__cluster.tpuIpv4CidrBlock }}", "verticalPodAutoscaling": "{{ .values.data__cluster.verticalPodAutoscaling }}", "workloadIdentityConfig": "{{ .values.data__cluster.workloadIdentityConfig }}", "zone": "{{ .values.data__cluster.zone }}" }', + '{ "addonsConfig": {{ .values.data__cluster.addonsConfig }}, "authenticatorGroupsConfig": {{ .values.data__cluster.authenticatorGroupsConfig }}, "autopilot": {{ .values.data__cluster.autopilot }}, "autoscaling": {{ .values.data__cluster.autoscaling }}, "binaryAuthorization": {{ .values.data__cluster.binaryAuthorization }}, "clusterIpv4Cidr": "{{ .values.data__cluster.clusterIpv4Cidr }}", "conditions": [ {{ .values.data__cluster.conditions[0] }} ], "confidentialNodes": {{ .values.data__cluster.confidentialNodes }}, "createTime": "{{ .values.data__cluster.createTime }}", "currentMasterVersion": "{{ .values.data__cluster.currentMasterVersion }}", "currentNodeCount": {{ .values.data__cluster.currentNodeCount }}, "currentNodeVersion": "{{ .values.data__cluster.currentNodeVersion }}", "databaseEncryption": {{ .values.data__cluster.databaseEncryption }}, "defaultMaxPodsConstraint": {{ .values.data__cluster.defaultMaxPodsConstraint }}, "description": "{{ .values.data__cluster.description }}", "enableKubernetesAlpha": {{ .values.data__cluster.enableKubernetesAlpha }}, "enableTpu": {{ .values.data__cluster.enableTpu }}, "endpoint": "{{ .values.data__cluster.endpoint }}", "expireTime": "{{ .values.data__cluster.expireTime }}", "initialClusterVersion": "{{ .values.data__cluster.initialClusterVersion }}", "initialNodeCount": {{ .values.data__cluster.initialNodeCount }}, "instanceGroupUrls": [ "{{ .values.data__cluster.instanceGroupUrls[0] }}" ], "ipAllocationPolicy": {{ .values.data__cluster.ipAllocationPolicy }}, "labelFingerprint": "{{ .values.data__cluster.labelFingerprint }}", "legacyAbac": {{ .values.data__cluster.legacyAbac }}, "location": "{{ .values.data__cluster.location }}", "locations": [ "{{ .values.data__cluster.locations[0] }}" ], "loggingConfig": {{ .values.data__cluster.loggingConfig }}, "loggingService": "{{ .values.data__cluster.loggingService }}", "maintenancePolicy": {{ .values.data__cluster.maintenancePolicy }}, "masterAuth": {{ .values.data__cluster.masterAuth }}, "masterAuthorizedNetworksConfig": {{ .values.data__cluster.masterAuthorizedNetworksConfig }}, "meshCertificates": {{ .values.data__cluster.meshCertificates }}, "monitoringConfig": {{ .values.data__cluster.monitoringConfig }}, "monitoringService": "{{ .values.data__cluster.monitoringService }}", "name": "{{ .values.data__cluster.name }}", "network": "{{ .values.data__cluster.network }}", "networkConfig": {{ .values.data__cluster.networkConfig }}, "networkPolicy": {{ .values.data__cluster.networkPolicy }}, "nodeConfig": {{ .values.data__cluster.nodeConfig }}, "nodeIpv4CidrSize": {{ .values.data__cluster.nodeIpv4CidrSize }}, "nodePoolDefaults": {{ .values.data__cluster.nodePoolDefaults }}, "nodePools": [ {{ .values.data__cluster.nodePools[0] }} ], "notificationConfig": {{ .values.data__cluster.notificationConfig }}, "privateClusterConfig": {{ .values.data__cluster.privateClusterConfig }}, "releaseChannel": {{ .values.data__cluster.releaseChannel }}, "resourceLabels": {{ .values.data__cluster.resourceLabels }}, "resourceUsageExportConfig": {{ .values.data__cluster.resourceUsageExportConfig }}, "selfLink": "{{ .values.data__cluster.selfLink }}", "servicesIpv4Cidr": "{{ .values.data__cluster.servicesIpv4Cidr }}", "shieldedNodes": {{ .values.data__cluster.shieldedNodes }}, "status": "{{ .values.data__cluster.status }}", "statusMessage": "{{ .values.data__cluster.statusMessage }}", "subnetwork": "{{ .values.data__cluster.subnetwork }}", "tpuIpv4CidrBlock": "{{ .values.data__cluster.tpuIpv4CidrBlock }}", "verticalPodAutoscaling": {{ .values.data__cluster.verticalPodAutoscaling }}, "workloadIdentityConfig": {{ .values.data__cluster.workloadIdentityConfig }}, "zone": "{{ .values.data__cluster.zone }}" }', '{{ .values.data__parent }}', '{{ .values.data__projectId }}', '{{ .values.data__zone }}' -; \ No newline at end of file +; diff --git a/test/assets/expected/views/select-cross-cloud-disks.json b/test/assets/expected/views/select-cross-cloud-disks.json index 5ff1b1271..218b9b5f5 100644 --- a/test/assets/expected/views/select-cross-cloud-disks.json +++ b/test/assets/expected/views/select-cross-cloud-disks.json @@ -1,8 +1,8 @@ [ - { "vendor": "aws", "name": "vol-00200000000000000", "status": "available", "type": "gp2", "size": "8" }, - { "vendor": "aws", "name": "vol-00100000000000000", "status": "available", "type": "gp2", "size": "10" }, - { "vendor": "google", "name": "my-disk-01-indeed", "status": "READY", "type": "standard", "size": "10" }, - { "vendor": "google", "name": "instance-1-c", "status": "READY", "type": "standard", "size": "10" }, - { "vendor": "google", "name": "instance-1-b", "status": "READY", "type": "standard", "size": "10" }, - { "vendor": "google", "name": "instance-1", "status": "READY", "type": "balanced", "size": "10" } + { "vendor": "aws", "name": "vol-00200000000000000", "status": "available", "type": "gp2", "size": 8 }, + { "vendor": "aws", "name": "vol-00100000000000000", "status": "available", "type": "gp2", "size": 10 }, + { "vendor": "google", "name": "my-disk-01-indeed", "status": "READY", "type": "standard", "size": 10 }, + { "vendor": "google", "name": "instance-1-c", "status": "READY", "type": "standard", "size": 10 }, + { "vendor": "google", "name": "instance-1-b", "status": "READY", "type": "standard", "size": 10 }, + { "vendor": "google", "name": "instance-1", "status": "READY", "type": "balanced", "size": 10 } ] diff --git a/test/python/stackql_test_tooling/StackQLInterfaces.py b/test/python/stackql_test_tooling/StackQLInterfaces.py index 3ffb0d555..cf4689df7 100644 --- a/test/python/stackql_test_tooling/StackQLInterfaces.py +++ b/test/python/stackql_test_tooling/StackQLInterfaces.py @@ -1234,11 +1234,6 @@ def should_stackql_exec_inline_equal_both_streams( **cfg ): comparison = {'unordered_http_diagnostics': cfg.pop('unordered_http_diagnostics', False)} - omni_expected = cfg.pop('omni_expected_stderr_output', None) - effective_preview = next((arg.split('=', 1)[1] for arg in _with_default_preview(args) - if isinstance(arg, str) and arg.startswith('--preview=')), '{}') - if omni_expected is not None and json.loads(effective_preview).get('omni') == 'all': - expected_stderr_output = omni_expected repeat_count = int(cfg.pop('repeat_count', 1)) for _ in range(repeat_count): result = self._run_stackql_exec_command( diff --git a/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json b/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json index f83300840..ea5ef3117 100644 --- a/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json +++ b/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json @@ -2203,7 +2203,7 @@ "Version": [ "^2016\\-11\\-15$" ], - "InstanceId": [ + "InstanceId.1": [ "id-001" ] } diff --git a/test/python/stackql_test_tooling/flask/gcp/app.py b/test/python/stackql_test_tooling/flask/gcp/app.py index f2bb06814..97ccb028a 100644 --- a/test/python/stackql_test_tooling/flask/gcp/app.py +++ b/test/python/stackql_test_tooling/flask/gcp/app.py @@ -412,6 +412,11 @@ def projects_testing_project_global_firewalls(): # Initialize a counter to track the number of calls call_counter = {"firewalls": 0} +@app.route('/reset/changing-firewalls', methods=['POST']) +def reset_changing_firewalls(): + call_counter["firewalls"] = 0 + return '', 204 + @app.route('/compute/v1/projects/changing-project/global/firewalls', methods=['GET']) def firewalls(): # Increment the call counter diff --git a/test/python/stackql_test_tooling/flask/native_test/app.py b/test/python/stackql_test_tooling/flask/native_test/app.py index 704654067..6962ec54a 100644 --- a/test/python/stackql_test_tooling/flask/native_test/app.py +++ b/test/python/stackql_test_tooling/flask/native_test/app.py @@ -136,6 +136,22 @@ def casing_echo_strict_get(): def casing_echo_filter_get(): return jsonify({"echoed_query": request.query_string.decode("utf-8")}) + @app.get("/casing/access_keys") + def casing_access_keys(): + return jsonify(items=[ + {"UserName": "alice", "AccessKeyId": "key-recent", "Status": "Active"}, + {"UserName": "alice", "AccessKeyId": "key-never", "Status": "Active"}, + {"UserName": "alice", "AccessKeyId": "key-inactive", "Status": "Inactive"}, + ]) + + @app.get("/casing/access_key_last_used") + def casing_access_key_last_used(): + from datetime import datetime, timezone + used = None + if request.args.get("AccessKeyId") == "key-recent": + used = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + return jsonify(LastUsedDate=used, Region="eu-west-1") + # ---- OCI request signing (oci_signing_v1) target ----------------------- @app.get("/oci/buckets") diff --git a/test/python/stackql_test_tooling/registry_rewrite.py b/test/python/stackql_test_tooling/registry_rewrite.py index 6cbdbb4cd..52baf025f 100644 --- a/test/python/stackql_test_tooling/registry_rewrite.py +++ b/test/python/stackql_test_tooling/registry_rewrite.py @@ -41,7 +41,7 @@ parser.add_argument( '--replacement-host', type=str, - default='localhost', + default='127.0.0.1', help='host name to overwrite in docs' ) parser.add_argument( @@ -125,7 +125,7 @@ class ProviderArgs: - def __init__(self, name :str, srcdir :str, destdir :str, port :int, replacement_host :str='localhost'): + def __init__(self, name :str, srcdir :str, destdir :str, port :int, replacement_host :str='127.0.0.1'): self.name = name self.srcdir = srcdir self.destdir = destdir diff --git a/test/python/stackql_test_tooling/stackql_context.py b/test/python/stackql_test_tooling/stackql_context.py index 80035d053..e54512a15 100644 --- a/test/python/stackql_test_tooling/stackql_context.py +++ b/test/python/stackql_test_tooling/stackql_context.py @@ -643,8 +643,6 @@ def generate_password() -> str: SELECT_CONTAINER_SUBNET_AGG_DESC = "select ipCidrRange, sum(5) cc from google.container.\"projects.aggregated.usableSubnetworks\" where projectsId = 'testing-project' group by ipCidrRange having sum(5) >= 5 order by ipCidrRange desc;" SELECT_CONTAINER_SUBNET_AGG_ASC = "select ipCidrRange, sum(5) cc from google.container.\"projects.aggregated.usableSubnetworks\" where projectsId = 'testing-project' group by ipCidrRange having sum(5) >= 5 order by ipCidrRange asc;" SELECT_ACCELERATOR_TYPES_DESC = "select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'testing-project' and zone = 'australia-southeast1-a' order by name desc;" - SELECT_ACCELERATOR_TYPES_DESC_FROM_INTEL_VIEWS = "select kind, name from stackql_intel.\"google.compute.acceleratorTypes\" where project = 'testing-project' and zone like '%%australia-southeast1-a' order by name desc;" - SELECT_ACCELERATOR_TYPES_DESC_FROM_INTEL_VIEWS_SUBQUERY = "SELECT name AS name, count(kind) AS \"COUNT(kind)\" FROM (SELECT * from stackql_intel.\"google.compute.acceleratorTypes\" limit 80) AS virtual_table GROUP BY name ORDER BY \"COUNT(kind)\" DESC LIMIT 1000;" SELECT_MACHINE_TYPES_DESC = "select name from google.compute.machineTypes where project = 'testing-project' and zone = 'australia-southeast1-a' order by name desc;" SELECT_GOOGLE_COMPUTE_INSTANCE_IAM_POLICY = "SELECT etag FROM google.compute.instances_iam_policies WHERE project = 'testing-project' AND zone = 'australia-southeast1-a' AND resource = '000000001';" @@ -894,8 +892,13 @@ def get_registry_mock_url(execution_env :str) -> str: print(f'sundry_config: "{sundry_config}"') _sundry_config = json.loads(sundry_config) if sundry_config else {} must_use_stackql_preinstalled :bool = use_stackql_preinstalled.lower() == 'true' - NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_ONE = get_native_query_row_count_from_table('okta.application.apps.Application.generation_1', sql_backend_str) - NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_THREE = get_native_query_row_count_from_table('okta.application.apps.Application.generation_3', sql_backend_str) + # Query-owned Omni staging must be released before the next statement. + if sql_backend_str == 'postgres_tcp': + staging_count_sql = "SELECT COUNT(*) as object_count FROM information_schema.tables WHERE table_schema = 'stackql_queries'" + else: + staging_count_sql = "SELECT COUNT(*) as object_count FROM sqlite_master WHERE type = 'table' AND name GLOB '__iql__.queries.[0-9]*'" + NATIVEQUERY_OMNI_STAGING_TABLE_COUNT = "NATIVEQUERY '" + staging_count_sql.replace("'", "''") + "';" + rv = { ## general config 'AUTH_AZURE_FAULTY': '{ "azure": { "type": "service_account", "credentialsenvar": "SOME_RUBBISH_ENV_VAR" } }', @@ -1008,7 +1011,6 @@ def get_registry_mock_url(execution_env :str) -> str: 'REGISTRY_LIST_EXPECTED': REGISTRY_LIST_EXPECTED, 'SELECT_ACCELERATOR_TYPES_DESC': SELECT_ACCELERATOR_TYPES_DESC, 'SELECT_ACCELERATOR_TYPES_DESC_EXPECTED': SELECT_ACCELERATOR_TYPES_DESC_EXPECTED, - 'SELECT_ACCELERATOR_TYPES_DESC_SEQUENCE': [ SELECT_ACCELERATOR_TYPES_DESC, SELECT_ACCELERATOR_TYPES_DESC_FROM_INTEL_VIEWS, SELECT_ACCELERATOR_TYPES_DESC_FROM_INTEL_VIEWS_SUBQUERY ], 'SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_EXPECTED': SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_EXPECTED, 'SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_SIMPLE': SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_SIMPLE, 'SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_TRANSPARENT': SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_TRANSPARENT, @@ -1113,9 +1115,9 @@ def get_registry_mock_url(execution_env :str) -> str: 'SHELL_COMMANDS_DISKS_VIEW_ALIASED_SEQUENCE_JSON_EXPECTED': SELECT_CROSS_CLOUD_DISKS_VIEW_EXPECTED_JSON, 'SHELL_COMMANDS_DISKS_VIEW_NOT_ALIASED_SEQUENCE': [ CREATE_DISKS_VIEW_NO_PRIMARY_ALIAS, "select * from cross_cloud_disks_not_aliased order by name desc;", "drop view cross_cloud_disks_not_aliased;" ], 'SHELL_COMMANDS_DISKS_VIEW_NOT_ALIASED_SEQUENCE_JSON_EXPECTED': SELECT_CROSS_CLOUD_DISKS_VIEW_EXPECTED_JSON, - 'SHELL_COMMANDS_GC_SEQUENCE_CANONICAL': [ SELECT_OKTA_APPS, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_THREE, PURGE_CONSERVATIVE, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_THREE, SELECT_OKTA_APPS, SELECT_OKTA_APPS, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_THREE, PURGE_CONSERVATIVE, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_THREE ], - 'SHELL_COMMANDS_GC_SEQUENCE_CANONICAL_JSON_EXPECTED': SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(5)] + PURGE_CONSERVATIVE_RESPONSE_JSON + [ get_object_count_dict(0) ] + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(10)] + PURGE_CONSERVATIVE_RESPONSE_JSON + [get_object_count_dict(0) ], - 'SHELL_COMMANDS_GC_SEQUENCE_EAGER': [ SELECT_OKTA_APPS, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_ONE, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_ONE, SELECT_OKTA_APPS, SELECT_OKTA_APPS, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_ONE, NATIVEQUERY_OKTA_APPS_ROW_COUNT_DISCO_ID_ONE ], + 'SHELL_COMMANDS_GC_SEQUENCE_CANONICAL': [ SELECT_OKTA_APPS, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, PURGE_CONSERVATIVE, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, SELECT_OKTA_APPS, SELECT_OKTA_APPS, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, PURGE_CONSERVATIVE, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT ], + 'SHELL_COMMANDS_GC_SEQUENCE_CANONICAL_JSON_EXPECTED': SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(0)] + PURGE_CONSERVATIVE_RESPONSE_JSON + [ get_object_count_dict(0) ] + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(0)] + PURGE_CONSERVATIVE_RESPONSE_JSON + [get_object_count_dict(0) ], + 'SHELL_COMMANDS_GC_SEQUENCE_EAGER': [ SELECT_OKTA_APPS, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, SELECT_OKTA_APPS, SELECT_OKTA_APPS, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT, NATIVEQUERY_OMNI_STAGING_TABLE_COUNT ], 'SHELL_COMMANDS_GC_SEQUENCE_EAGER_JSON_EXPECTED': SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(0)] + [ get_object_count_dict(0) ] + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + SELECT_OKTA_APPS_ASC_EXPECTED_JSON + [ get_object_count_dict(0)] + [get_object_count_dict(0) ], 'SHELL_COMMANDS_SPECIALCASE_REPEATED_CACHED': [ SELECT_GITHUB_JOIN_IN_PARAMS_SPECIALCASE, SELECT_GITHUB_JOIN_IN_PARAMS_SPECIALCASE ], 'SHELL_COMMANDS_SPECIALCASE_REPEATED_CACHED_JSON_EXPECTED': SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_SPECIALCASE_JSON_EXPECTED + SELECT_ANALYTICS_CACHE_GITHUB_REPOSITORIES_COLLABORATORS_SPECIALCASE_JSON_EXPECTED, @@ -1200,7 +1202,7 @@ def get_registry_mock_url(execution_env :str) -> str: else: rv['AUTH_CFG_STR'] = AUTH_CFG_STR rv['AUTH_CFG_DEFECTIVE_STR'] = AUTH_CFG_DEFECTIVE_STR - rv['LOCAL_HOST_ALIAS'] = 'localhost' + rv['LOCAL_HOST_ALIAS'] = '127.0.0.1' rv['AUTH_GOOGLE_SA_KEY_PATH'] = _AUTH_GOOGLE_SA_KEY_PATH rv['NON_EXISTENT_AUTH_GOOGLE_SA_KEY_PATH'] = _NON_EXISTENT_AUTH_GOOGLE_SA_KEY_PATH rv['AUTH_PLUS_EXTERNAL_POSTGRES'] = AUTH_PLUS_EXTERNAL_POSTGRES diff --git a/test/python/stackql_test_tooling/typed_python_responses.py b/test/python/stackql_test_tooling/typed_python_responses.py index ba9ebe19c..d645aca7a 100644 --- a/test/python/stackql_test_tooling/typed_python_responses.py +++ b/test/python/stackql_test_tooling/typed_python_responses.py @@ -2,37 +2,14 @@ SELECT_AWS_CLOUD_CONTROL_EVENTS_MINIMAL_EXPECTED = [ - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, - {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, {'EventTime': decimal.Decimal('1660709730.739'), 'Identifier': None }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample0' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample1' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample1' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample2' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample2' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample3' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample3' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample4' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample4' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample5' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample5' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample6' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample6' }, - {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample7' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample7' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample8' }, {'EventTime': decimal.Decimal('1660266055.925'), 'Identifier': 'CloudControlExample9' }, @@ -44,35 +21,21 @@ {'EventTime': decimal.Decimal('1660709644.835'), 'Identifier': 'LogGroupResourceExample9' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond0' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond1' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond1' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond2' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond2' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond3' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond3' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond4' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond4' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond5' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond5' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond6' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond6' }, - {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond7' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond7' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond8' }, {'EventTime': decimal.Decimal('1660709766.092'), 'Identifier': 'LogGroupResourceExampleSecond9' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird0' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird1' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird1' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird2' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird2' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird3' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird3' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird4' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird4' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird5' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird5' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird6' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird6' }, - {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird7' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird7' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird8' }, {'EventTime': decimal.Decimal('1660714274.285'), 'Identifier': 'LogGroupResourceExampleThird9' } diff --git a/test/python/stackql_test_tooling/web_service_keywords.py b/test/python/stackql_test_tooling/web_service_keywords.py index d83ccfef0..a780c6574 100644 --- a/test/python/stackql_test_tooling/web_service_keywords.py +++ b/test/python/stackql_test_tooling/web_service_keywords.py @@ -6,6 +6,7 @@ from requests import get, post, Response import os +import sys from pathlib import Path, PurePosixPath @@ -435,6 +436,23 @@ def create_digitalocean_web_service( cwd=self._cwd, ) + @keyword + def reset_changing_firewalls(self, execution_platform: str, port: int = 1080) -> None: + # Reset inside the mock container for Docker execution. Provider-facing + # host aliases are not addresses the Robot runner can necessarily resolve. + command = [sys.executable] + if execution_platform == 'docker': + command = ['docker', 'compose', '-f', 'docker-compose-testing.yml', + 'exec', '-T', 'google_mockserver', 'python'] + program = ( + "import ssl; from urllib.request import Request, urlopen; " + f"urlopen(Request('https://127.0.0.1:{port}/reset/changing-firewalls', " + "method='POST'), context=ssl._create_unverified_context(), timeout=10).read()" + ) + result = self.run_process(*command, '-c', program, cwd=self._cwd) + if result.rc != 0: + raise RuntimeError(f'Cannot reset changing-firewalls mock: {result.stderr}') + @keyword def start_all_webservers(self, port_dict: Optional[dict] = None) -> None: # if system has docker installed and also has docker compose file, use that to run mock servers diff --git a/test/registry/src/stackql_native_test/v0.1.0/services/casing.yaml b/test/registry/src/stackql_native_test/v0.1.0/services/casing.yaml index 3dac4b35f..6f549d21c 100644 --- a/test/registry/src/stackql_native_test/v0.1.0/services/casing.yaml +++ b/test/registry/src/stackql_native_test/v0.1.0/services/casing.yaml @@ -89,6 +89,26 @@ components: requestId: type: string x-stackQL-resources: + access_keys: + id: stackql_native_test.casing.access_keys + name: access_keys + methods: + select: + operation: {$ref: '#/paths/~1casing~1access_keys/get'} + request: {nativeCasing: pascal} + response: {mediaType: application/json, objectKey: $.items} + sqlVerbs: + select: [{$ref: '#/components/x-stackQL-resources/access_keys/methods/select'}] + access_key_last_useds: + id: stackql_native_test.casing.access_key_last_useds + name: access_key_last_useds + methods: + select: + operation: {$ref: '#/paths/~1casing~1access_key_last_used/get'} + request: {nativeCasing: pascal} + response: {mediaType: application/json, objectKey: $} + sqlVerbs: + select: [{$ref: '#/components/x-stackQL-resources/access_key_last_useds/methods/select'}] echo: id: stackql_native_test.casing.echo name: echo @@ -422,3 +442,38 @@ paths: servers: - url: http://localhost:1070 summary: Lifecycle action whose response omits the id + /casing/access_keys: + get: + parameters: + - {in: query, name: region, required: true, schema: {type: string}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + properties: + UserName: {type: string} + AccessKeyId: {type: string} + Status: {type: string} + /casing/access_key_last_used: + get: + parameters: + - {in: query, name: region, required: true, schema: {type: string}} + - {in: query, name: AccessKeyId, required: true, schema: {type: string}} + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + LastUsedDate: {type: string, nullable: true} + Region: {type: string} diff --git a/test/registry/src/stackql_test/v00.00.00000/provider.yaml b/test/registry/src/stackql_test/v00.00.00000/provider.yaml index 409db1200..cabec9d01 100644 --- a/test/registry/src/stackql_test/v00.00.00000/provider.yaml +++ b/test/registry/src/stackql_test/v00.00.00000/provider.yaml @@ -1,6 +1,6 @@ -id: azuread +id: stackql_test description: A testing provider, originating from Azure Active Directory -name: azuread +name: stackql_test version: v00.00.00000 providerServices: applications: diff --git a/test/robot/functional/stackql_body_coercion.robot b/test/robot/functional/stackql_body_coercion.robot index a9870293c..cfebd2012 100644 --- a/test/robot/functional/stackql_body_coercion.robot +++ b/test/robot/functional/stackql_body_coercion.robot @@ -100,7 +100,7 @@ Exec Json Payload Fractional Value For Integer Property Rejected ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... exec stackql_native_test.coerce.settings.update @serviceId \= 'svc-3' @@json \= '{"retryCount": 3.5}'; - ... key 'retryCount' expected to contain element of type + ... request body key 'retryCount': value '3.5' cannot be coerced to schema type 'integer' Update Unparseable String For Number Property Fails Before Request [Documentation] Negative: the coercion error names the wire key and schema type; diff --git a/test/robot/functional/stackql_casing.robot b/test/robot/functional/stackql_casing.robot index 6bb156417..627fdccd9 100644 --- a/test/robot/functional/stackql_casing.robot +++ b/test/robot/functional/stackql_casing.robot @@ -400,8 +400,8 @@ Wire Spelled Order By Sorts Snake Display Column Unknown Snake Projection Still Rejected [Documentation] Negative: a column absent from the schema is not conjured by the - ... display-name re-keying; the backend rejects it in its own dialect. - ${expected} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" column "vpc_idz" does not exist no such column: vpc_idz + ... display-name re-keying; Omni rejects the unknown column before backend execution. + ${expected} = Set Variable no such column: vpc_idz Should Stackql Exec Inline Contain Stderr ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -450,3 +450,36 @@ Backward Compat Wire Exec Argument ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... select binary_id, echoed_query from (exec stackql_native_test.casing.echo_bodyless.exec_by_id @BinaryId \= 'bw', @VpcId \= 'vw'); ... VpcId\=vw + +Access Key Audit Join Keeps Request Region Separate From Response Region + ${query} = Catenate + ... SELECT strftime('%Y-%m-%dT%H:%M:%SZ', 'now') AS observed_at, 'aws_access_key_unused' AS check_id, + ... 'aws' AS provider, '111111111111' AS tenant, 'global' AS region, + ... k.user_name || '/' || k.access_key_id AS asset, 'access key last used' AS attribute, + ... COALESCE(l.last_used_date, 'never') AS observed, + ... CASE WHEN l.last_used_date >= datetime('now', '-90 days') THEN 'pass' ELSE 'fail' END AS result + ... FROM stackql_native_test.casing.access_keys k + ... INNER JOIN stackql_native_test.casing.access_key_last_useds l + ... ON l.AccessKeyId = k.AccessKeyId AND l.region = 'us-east-1' + ... WHERE k.region = 'us-east-1' AND k.status = 'Active'; + ${output} = Run Stackql Exec Command No Errors + ... ${STACKQL_EXE} ${OKTA_SECRET_STR} ${GITHUB_SECRET_STR} ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} ${AUTH_CFG_STR} ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} stackql_output=csv + Should Be Empty ${output.stderr} + ${rows} = Evaluate list(csv.DictReader(io.StringIO($output.stdout))) modules=csv,io + Length Should Be ${rows} 2 + ${results} = Evaluate {row['asset']: row['result'] for row in $rows} + ${expected} = Evaluate {'alice/key-recent': 'pass', 'alice/key-never': 'fail'} + Should Be Equal ${results} ${expected} + ${unused} = Evaluate next(row for row in $rows if row['asset'] == 'alice/key-never') + Should Be Equal ${unused}[observed] never + FOR ${row} IN @{rows} + Should Be Equal ${row}[region] global + Should Match Regexp ${row}[observed_at] ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + END + Should Stackql Exec Inline Contain + ... ${STACKQL_EXE} ${OKTA_SECRET_STR} ${GITHUB_SECRET_STR} ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} ${AUTH_CFG_STR} ${SQL_BACKEND_CFG_STR_CANONICAL} + ... select Region, region from stackql_native_test.casing.access_key_last_useds where region \= 'us-east-1' and AccessKeyId \= 'key-recent'; + ... eu-west-1 diff --git a/test/robot/functional/stackql_from_cmd_line.robot b/test/robot/functional/stackql_from_cmd_line.robot index b69eeb058..930d2f711 100644 --- a/test/robot/functional/stackql_from_cmd_line.robot +++ b/test/robot/functional/stackql_from_cmd_line.robot @@ -91,11 +91,9 @@ Describe allOf Object Returns Expected Result ... |---------------------------------------|---------| ... |${SPACE}@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| ... |---------------------------------------|---------| - ... |${SPACE}_@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |---------------------------------------|---------| ... |${SPACE}aboutMe${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| ... |---------------------------------------|---------| - ... |${SPACE}accountEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| + ... |${SPACE}accountEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| ... |${SPACE}activities${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| @@ -201,7 +199,7 @@ Describe allOf Object Returns Expected Result ... |---------------------------------------|---------| ... |${SPACE}interests${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| - ... |${SPACE}isResourceAccount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| + ... |${SPACE}isResourceAccount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| ... |${SPACE}jobTitle${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| ... |---------------------------------------|---------| @@ -257,7 +255,7 @@ Describe allOf Object Returns Expected Result ... |---------------------------------------|---------| ... |${SPACE}onPremisesSecurityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| ... |---------------------------------------|---------| - ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| + ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| ... |${SPACE}onPremisesUserPrincipalName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| ... |---------------------------------------|---------| @@ -317,7 +315,7 @@ Describe allOf Object Returns Expected Result ... |---------------------------------------|---------| ... |${SPACE}settings${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| ... |---------------------------------------|---------| - ... |${SPACE}showInAddressList${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| + ... |${SPACE}showInAddressList${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| ... |---------------------------------------|---------| ... |${SPACE}signInActivity${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| ... |---------------------------------------|---------| @@ -351,167 +349,161 @@ Describe allOf Object Returns Expected Result Describe anyOf Object Returns Expected Result ${outputStr} = Catenate SEPARATOR=\n - ... |--------------------------------------|---------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}description${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}_@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}acceptedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}address${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}allowExternalSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}appRoleAssignments${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}assignedLabels${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}assignedLicenses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}autoSubscribeNewMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}calendar${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}calendarView${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}classification${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}conversations${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}coordinates${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}createdDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}createdOnBehalfOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}deletedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}displayName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}drive${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}drives${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}events${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}expirationDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}extensions${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}groupLifecyclePolicies${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}groupTypes${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hasMembersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hideFromAddressLists${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hideFromOutlookClients${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isArchived${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isAssignableToRole${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isSubscribedByMail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}licenseProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationEmailAddress${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationUri${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mailEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mailNickname${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}memberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}members${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membershipRule${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membershipRuleProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}microsoft.graph.location_@odata.type${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}microsoft.graph.location_displayName${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesDomainName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesLastSyncDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesNetBiosName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSamAccountName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSecurityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onenote${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}owners${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}permissionGrants${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}photo${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}photos${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}planner${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}preferredDataLocation${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}preferredLanguage${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}proxyAddresses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}rejectedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}renewedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}securityEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}securityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}serviceProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}settings${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}sites${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}team${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}theme${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}threads${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}transitiveMemberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}transitiveMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}uniqueId${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}uniqueIdType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}unseenCount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}integer${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}visibility${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| + ... |-------------------------------|---------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}description${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}acceptedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}address${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}allowExternalSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}appRoleAssignments${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}assignedLabels${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}assignedLicenses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}autoSubscribeNewMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}calendar${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}calendarView${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}classification${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}conversations${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}coordinates${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}createdDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}createdOnBehalfOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}deletedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}displayName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}drive${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}drives${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}events${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}expirationDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}extensions${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}groupLifecyclePolicies${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}groupTypes${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hasMembersWithLicenseErrors${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hideFromAddressLists${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hideFromOutlookClients${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isArchived${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isAssignableToRole${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isSubscribedByMail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}licenseProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationEmailAddress${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}text${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationUri${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mailEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mailNickname${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}memberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}members${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membershipRule${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membershipRuleProcessingState${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesDomainName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesLastSyncDateTime${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesNetBiosName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesProvisioningErrors${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSamAccountName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSecurityIdentifier${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onenote${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}owners${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}permissionGrants${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}photo${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}photos${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}planner${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}preferredDataLocation${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}preferredLanguage${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}proxyAddresses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}rejectedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}renewedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}securityEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}securityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}serviceProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}settings${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}sites${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}team${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}theme${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}threads${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}transitiveMemberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}transitiveMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}uniqueId${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}uniqueIdType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}text${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}unseenCount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}integer${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}visibility${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| Should StackQL Current Exec Equal ... describe stackql_test.totally_contrived.any_of_explore; ... ${outputStr} @@ -520,167 +512,161 @@ Describe anyOf Object Returns Expected Result Describe oneOf Object Returns Expected Result ${outputStr} = Catenate SEPARATOR=\n - ... |--------------------------------------|---------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}description${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}_@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}acceptedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}address${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}allowExternalSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}appRoleAssignments${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}assignedLabels${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}assignedLicenses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}autoSubscribeNewMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}calendar${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}calendarView${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}classification${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}conversations${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}coordinates${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}createdDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}createdOnBehalfOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}deletedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}displayName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}drive${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}drives${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}events${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}expirationDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}extensions${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}groupLifecyclePolicies${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}groupTypes${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hasMembersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hideFromAddressLists${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}hideFromOutlookClients${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isArchived${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isAssignableToRole${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}isSubscribedByMail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}licenseProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationEmailAddress${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}locationUri${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mailEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}mailNickname${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}memberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}members${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membershipRule${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}membershipRuleProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}microsoft.graph.location_@odata.type${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}microsoft.graph.location_displayName${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesDomainName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesLastSyncDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesNetBiosName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSamAccountName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSecurityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}onenote${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}owners${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}permissionGrants${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}photo${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}photos${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}planner${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}preferredDataLocation${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}preferredLanguage${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}proxyAddresses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}rejectedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}renewedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}securityEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}boolean${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}securityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}serviceProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}settings${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}sites${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}team${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}theme${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}threads${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}transitiveMemberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}transitiveMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}uniqueId${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}uniqueIdType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}unseenCount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}integer${SPACE}| - ... |--------------------------------------|---------| - ... |${SPACE}visibility${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| - ... |--------------------------------------|---------| + ... |-------------------------------|---------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}description${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}@odata.type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}acceptedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}address${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}allowExternalSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}appRoleAssignments${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}assignedLabels${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}assignedLicenses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}autoSubscribeNewMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}calendar${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}calendarView${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}classification${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}conversations${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}coordinates${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}createdDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}createdOnBehalfOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}deletedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}displayName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}drive${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}drives${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}events${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}expirationDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}extensions${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}groupLifecyclePolicies${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}groupTypes${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hasMembersWithLicenseErrors${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hideFromAddressLists${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}hideFromOutlookClients${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isArchived${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isAssignableToRole${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}isSubscribedByMail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}licenseProcessingState${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationEmailAddress${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}text${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}locationUri${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mail${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mailEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}mailNickname${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}memberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}members${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membersWithLicenseErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membershipRule${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}membershipRuleProcessingState${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesDomainName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesLastSyncDateTime${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesNetBiosName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesProvisioningErrors${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSamAccountName${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSecurityIdentifier${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onPremisesSyncEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}onenote${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}owners${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}permissionGrants${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}photo${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}photos${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}planner${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}preferredDataLocation${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}preferredLanguage${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}proxyAddresses${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}rejectedSenders${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}renewedDateTime${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}securityEnabled${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}bool${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}securityIdentifier${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}serviceProvisioningErrors${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}settings${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}sites${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}team${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}object${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}theme${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}threads${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}transitiveMemberOf${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}transitiveMembers${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}array${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}uniqueId${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}uniqueIdType${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}text${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}unseenCount${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}integer${SPACE}| + ... |-------------------------------|---------| + ... |${SPACE}visibility${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}string${SPACE}${SPACE}| + ... |-------------------------------|---------| Should StackQL Current Exec Equal ... describe stackql_test.totally_contrived.one_of_explore; ... ${outputStr} diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index d77c3ddb5..dee702324 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -1786,7 +1786,7 @@ Debug HTTP Plus Transparent Placeholder URL and Defaulted Request Body Returns E ... |------------|-------------| ... |${SPACE}bkp-000002${SPACE}|${SPACE}READY${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| ... |------------|-------------| - ${hostName} = Set Variable If "${EXECUTION_PLATFORM}" == "docker" host.docker.internal localhost + ${hostName} = Set Variable ${LOCAL_HOST_ALIAS} ${stderrStr} = Catenate SEPARATOR=\n ... http request url: 'https://${hostName}:1091/', method: 'POST' ... http request body = '{"Filters":{}}' @@ -1937,6 +1937,7 @@ Response Error Printed by Default on 403 Null Body Error ... stderr=${CURDIR}/tmp/Response-Error-Printed-by-Default-on-403-Error-stderr.tmp Create Changing Dynamic Materialized View Scenario Working + Reset Changing Firewalls ${EXECUTION_PLATFORM} ${MOCKSERVER_PORT_GOOGLE} ${inputStr} = Catenate ... create materialized view silly_changing_mv as select * from google.compute.firewalls where project = 'changing-project'; ... select name, id from silly_changing_mv order by name desc, id desc; @@ -3038,31 +3039,31 @@ Registry Pull Overwrite Works in Both Directions ... registry pull google 'v0.1.2'; ... show resources in google.storage; ${stdOutStr} = Catenate SEPARATOR=\n - ... |------|------------------------| - ... |${SPACE}name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------|------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}google.storage.buckets${SPACE}| - ... |------|------------------------| - ... |------|-------------------------------| - ... |${SPACE}name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------|-------------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}google.storage.bucket_objects${SPACE}| - ... |------|-------------------------------| - ... |------|--------------------------------| - ... |${SPACE}name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------|--------------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}google.storage.storage_buckets${SPACE}| - ... |------|--------------------------------| - ... |------|-------------------------------| - ... |${SPACE}name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------|-------------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}google.storage.bucket_objects${SPACE}| - ... |------|-------------------------------| - ... |------|------------------------| - ... |${SPACE}name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------|------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}google.storage.buckets${SPACE}| - ... |------|------------------------| + ... |---------|------------------------| + ... |${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |---------|------------------------| + ... |${SPACE}buckets${SPACE}|${SPACE}google.storage.buckets${SPACE}| + ... |---------|------------------------| + ... |----------------|-------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |----------------|-------------------------------| + ... |${SPACE}bucket_objects${SPACE}|${SPACE}google.storage.bucket_objects${SPACE}| + ... |----------------|-------------------------------| + ... |-----------------|--------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-----------------|--------------------------------| + ... |${SPACE}storage_buckets${SPACE}|${SPACE}google.storage.storage_buckets${SPACE}| + ... |-----------------|--------------------------------| + ... |----------------|-------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |----------------|-------------------------------| + ... |${SPACE}bucket_objects${SPACE}|${SPACE}google.storage.bucket_objects${SPACE}| + ... |----------------|-------------------------------| + ... |---------|------------------------| + ... |${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |---------|------------------------| + ... |${SPACE}buckets${SPACE}|${SPACE}google.storage.buckets${SPACE}| + ... |---------|------------------------| ${stdErrStr} = Catenate SEPARATOR=\n ... google provider, version 'v0.1.2' successfully installed ... google provider, version 'v0.1.1-alpha01' successfully installed @@ -4563,7 +4564,7 @@ Inline Union Select Returns Expected Result ... ${REGISTRY_NO_VERIFY_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... SELECT 'some\-placeholder' as bucket_name, 'ap\-southeast\-2' as region UNION SELECT 'some\-other\-placeholder' as bucket_name, 'ap\-southeast\-2' as region; + ... SELECT 'some\-placeholder' as bucket_name, 'ap\-southeast\-2' as region UNION SELECT 'some\-other\-placeholder' as bucket_name, 'ap\-southeast\-2' as region ORDER BY bucket_name; ... ${outputStr} ... ${CURDIR}/tmp/Inline-Union-Select-Returns-Expected-Result.tmp @@ -4900,7 +4901,7 @@ Verify Data Flow Replication in ON Conditions Is Mitigated Using Example of Netw ... from google.compute.networks nw LEFT OUTER JOIN google.compute.subnetworks sn ... on lower(nw.name) = lower(split_part(sn.network, '/', 10)) and sn.project = split_part(nw.selfLink, '/', 7) ... where nw.project = 'testing-project' and sn.region = 'australia-southeast1' - ... group by network_name, subnetwork_name ; + ... group by network_name, subnetwork_name order by network_name, subnetwork_name; ${outputStr} = Catenate SEPARATOR=\n ... |------------------------------|-----------------|--------------| ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}network_name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnetwork_name${SPACE}|${SPACE}subnet_count${SPACE}| @@ -6480,7 +6481,7 @@ Aggregation of View of Table Valued Function List And Detail Dataflow Works As E ... WHERE listing.data__TypeName = 'AWS::S3::Bucket' ... AND detail.data__TypeName = 'AWS::S3::Bucket' ... AND listing.region IN ('us-east-1', 'ap-southeast-1') - ... ; + ... order by id; ... select ... id, ... count(1) as total_tag_count @@ -6489,7 +6490,7 @@ Aggregation of View of Table Valued Function List And Detail Dataflow Works As E ... having json_extract(json_group_object(tag_key, tag_value), '$.provisioner') = 'stackql' ... and json_extract(json_group_object(tag_key, tag_value), '$.env') = 'dev' ... and json_extract(json_group_object(tag_key, tag_value), '$.stackid') = 'payruns' - ... ; + ... order by id; ... drop view if exists exotic_view_01; ${postgresInputStr} = Catenate ... create or replace view exotic_view_01 as @@ -6531,7 +6532,7 @@ Aggregation of View of Table Valued Function List And Detail Dataflow Works As E ... WHERE listing.data__TypeName = 'AWS::S3::Bucket' ... AND detail.data__TypeName = 'AWS::S3::Bucket' ... AND listing.region IN ('us-east-1', 'ap-southeast-1') - ... ; + ... order by id; ... select ... id, ... count(1) as total_tag_count @@ -6540,7 +6541,7 @@ Aggregation of View of Table Valued Function List And Detail Dataflow Works As E ... having json_extract_path_text(json_object_agg(tag_key, tag_value), 'provisioner') = 'stackql' ... and json_extract_path_text(json_object_agg(tag_key, tag_value), 'env') = 'dev' ... and json_extract_path_text(json_object_agg(tag_key, tag_value), 'stackid') = 'payruns' - ... ; + ... order by id; ... drop view if exists exotic_view_01; ${inputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${postgresInputStr} ${sqliteInputStr} ${outputStr} = Catenate SEPARATOR=\n @@ -6577,7 +6578,7 @@ Aggregation of Doc Based View of Table Valued Function List And Detail Dataflow ... having json_extract(json_group_object(tag_key, tag_value), '$.provisioner') = 'stackql' ... and json_extract(json_group_object(tag_key, tag_value), '$.env') = 'dev' ... and json_extract(json_group_object(tag_key, tag_value), '$.stackid') = 'payruns' - ... ; + ... order by id; ${postgresInputStr} = Catenate ... select ... id, @@ -6587,7 +6588,7 @@ Aggregation of Doc Based View of Table Valued Function List And Detail Dataflow ... having json_extract_path_text(json_object_agg(tag_key, tag_value), 'provisioner') = 'stackql' ... and json_extract_path_text(json_object_agg(tag_key, tag_value), 'env') = 'dev' ... and json_extract_path_text(json_object_agg(tag_key, tag_value), 'stackid') = 'payruns' - ... ; + ... order by id; ${inputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${postgresInputStr} ${sqliteInputStr} ${outputStr} = Catenate SEPARATOR=\n ... |-----------------------------|-----------------| @@ -6808,10 +6809,6 @@ Multi Dependency Multi Dependent Multiple List And Detail Dataflow Works As Exem ... |------------------|----------|-----------|----------|-------------------------------------------------------------|--------------------------------| ... |${SPACE}dummy-key-02${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}{}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}RSA${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}2048${SPACE}|${SPACE}\["sign","verify","wrapKey","unwrapKey","encrypt","decrypt"]${SPACE}|${SPACE}Microsoft.KeyVault/vaults/keys${SPACE}| ... |------------------|----------|-----------|----------|-------------------------------------------------------------|--------------------------------| - ${outputErrStr} = Catenate SEPARATOR=\n - ... http response status code: 404, response body is nil - ... http response status code: 404, response body is nil - ... http response status code: 404, response body is nil Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -6822,11 +6819,11 @@ Multi Dependency Multi Dependent Multiple List And Detail Dataflow Works As Exem ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${inputStr} ... ${outputStr} - ... ${outputErrStr} + ... ${EMPTY} ... stdout=${CURDIR}/tmp/Multi-Dependency-Multi-Dependent-Multiple-List-And-Detail-Dataflow-Works-As-Exemplified-By-Azure-Vault-and-Keys-and-Key-Details.tmp ... stderr=${CURDIR}/tmp/Multi-Dependency-Multi-Dependent-Multiple-List-And-Detail-Dataflow-Works-As-Exemplified-By-Azure-Vault-and-Keys-and-Key-Details-stderr.tmp ... stackql_dataflow_permissive=True - ... omni_expected_stderr_output=${EMPTY} + Error GTE 400 Response Code Does Not Stop The World As Exemplified By AWS Subnet Route Associations List And Detail Pattern ${sqliteInputStr} = Catenate @@ -7069,10 +7066,6 @@ Static Input Read Many Multi Dependency Multi Dependent Multiple List And Detail ... |------------------|----------|-----------|----------|-------------------------------------------------------------|--------------------------------| ... |${SPACE}dummy-key-02${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}{}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}RSA${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}2048${SPACE}|${SPACE}\["sign","verify","wrapKey","unwrapKey","encrypt","decrypt"]${SPACE}|${SPACE}Microsoft.KeyVault/vaults/keys${SPACE}| ... |------------------|----------|-----------|----------|-------------------------------------------------------------|--------------------------------| - ${outputErrStr} = Catenate SEPARATOR=\n - ... http response status code: 404, response body is nil - ... http response status code: 404, response body is nil - ... http response status code: 404, response body is nil Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -7083,10 +7076,10 @@ Static Input Read Many Multi Dependency Multi Dependent Multiple List And Detail ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${inputStr} ... ${outputStr} - ... ${outputErrStr} + ... ${EMPTY} ... stdout=${CURDIR}/tmp/Static-Input-Read-Many-Multi-Dependency-Multi-Dependent-Multiple-List-And-Detail-Dataflow-Works-As-Exemplified-By-Azure-Vault-and-Keys-and-Key-Details.tmp ... stderr=${CURDIR}/tmp/Static-Input-Read-Many-Multi-Dependency-Multi-Dependent-Multiple-List-And-Detail-Dataflow-Works-As-Exemplified-By-Azure-Vault-and-Keys-and-Key-Details-stderr.tmp - ... omni_expected_stderr_output=${EMPTY} + Self Join Polymorphic Works As Exemplified By Azure VPN List and Details ${sqliteInputStr} = Catenate @@ -8236,15 +8229,7 @@ Select Star From Transformed XML Response Body ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}0${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}0${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| - ... |${SPACE}availability_zone${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}encrypted${SPACE}|${SPACE}multi_attach_enabled${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}|${SPACE}snapshot_id${SPACE}|${SPACE}${SPACE}status${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}volume_type${SPACE}| - ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| - ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| - ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -8769,23 +8754,7 @@ Select Paginated Star From Flattened Transformed JSON Response Body ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet4${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}name${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet2${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet4${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| - ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -8819,23 +8788,7 @@ Select Paginated Projection From Flattened Transformed JSON Response Body ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -8885,35 +8838,7 @@ Select Join of Flattened Paginated Projection From Transformed JSON and XML Resp ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -8967,35 +8892,7 @@ Select View of Join of Flattened Paginated Projection From Transformed JSON and ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -9049,35 +8946,7 @@ Select Materialized View of Join of Flattened Paginated Projection From Transfor ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStrPostgres} = Catenate SEPARATOR=\n - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| - ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| - ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + ${outputStr} = Set Variable ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -9682,8 +9551,8 @@ Explain Select Repeatably Generates Messagez ... OK ... OK ... OK - ... cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS IN google.storage.buckets: no appropriate method = 'select' for resource = 'buckets' - ... cannot find matching operation, possible causes include missing required parameters or an unsupported method for the resource, to find required parameters for supported methods run SHOW METHODS IN google.storage.buckets: no appropriate method = 'select' for resource = 'buckets' + ... omnisdk: buckets (stackql_unstable_google.storage.buckets): no satisfiable select method (get needs bucket; list needs project) + ... omnisdk: buckets (stackql_unstable_google.storage.buckets): no satisfiable select method (get needs bucket; list needs project) ... Execution plan generated successfully ... OK ... Execution plan generated successfully @@ -10896,7 +10765,7 @@ Recursive CTE Is Not Supported At Execution ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${query} - ... cannot resolve ServiceHandle + ... recursive provider CTEs require an iterative execution boundary # =========================================================================== # Issue #700: a required structured parameter supplied as JSON() in a SELECT @@ -11643,14 +11512,14 @@ Unstable Github Org Members Staged Group By Exact Match ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Staged-Group-By.tmp ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Staged-Group-By-stderr.tmp -Omni All Github Org Members Staged Order By With Offset Exact Match - [Documentation] With omni set to all, a canonical provider name is routed - ... to omnisdk rather than any-sdk, and staging applies the +Omni Default Github Org Members Staged Order By With Offset Exact Match + [Documentation] A canonical provider name uses omnisdk by default, + ... and staging applies the ... ORDER BY, LIMIT and OFFSET. [Teardown] Remove Preview Mock Environment ${preview} = Catenate SEPARATOR= ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", - ... "insecureSkipTLSVerify":true,"omni":"all","staging":true} + ... "insecureSkipTLSVerify":true} ${query} = Catenate SEPARATOR=${SPACE} ... select login, type from github.orgs.members ... where org = 'dummyorg' order by login desc limit 3 offset 1; @@ -12260,8 +12129,8 @@ Preview Iac Destroy Without Collection Is Refused ... stdout=${CURDIR}${/}tmp${/}Preview-Iac-Destroy-No-Collection.tmp ... stderr=${CURDIR}${/}tmp${/}Preview-Iac-Destroy-No-Collection-stderr.tmp -Omni Derived Relation Preserves Inner Order And Limit - [Documentation] Derived relations execute through analyzer-owned Omni read handles. +Omni Default Derived Relation Preserves Inner Order And Limit + [Documentation] Derived relations execute through Omni read handles with no preview opt-in. ${query} = Catenate SEPARATOR=${SPACE} ... select count(*) as n from (select login, contributions from github.repos.contributors where owner = 'dummyorg' and repo = 'dummyapp.io' order by contributions desc limit 2) contributors; Should StackQL Exec Inline Equal @@ -12275,10 +12144,9 @@ Omni Derived Relation Preserves Inner Order And Limit ... ${query} ... n\n2 ... \-o\=csv - ... --preview\={"unstable":true,"staging":true,"omni":"all"} Omni Nested Derived Relation Preserves Aggregate Boundary - [Documentation] Derived relations execute through analyzer-owned Omni read handles. + [Documentation] Derived relations execute through Omni read handles with no preview opt-in. ${query} = Catenate SEPARATOR=${SPACE} ... select max(n) as n from (select count(*) as n from (select login from github.repos.contributors where owner = 'dummyorg' and repo = 'dummyapp.io' limit 2) limited) counted; Should StackQL Exec Inline Equal @@ -12292,7 +12160,6 @@ Omni Nested Derived Relation Preserves Aggregate Boundary ... ${query} ... n\n2 ... \-o\=csv - ... --preview\={"unstable":true,"staging":true,"omni":"all"} Omni Stored View Joins Provider Through Read Handles ${query} = Catenate SEPARATOR=${SPACE} @@ -12310,7 +12177,6 @@ Omni Stored View Joins Provider Through Read Handles ... ${query} ... n\n2 ... \-o\=csv - ... --preview\={"unstable":true,"staging":true,"omni":"all"} Omni CTEs Preserve Ordered Boundaries And Column Names @@ -12333,7 +12199,6 @@ Omni CTEs Preserve Ordered Boundaries And Column Names ... ${inputStr} ... ${expectedStr} ... \-o\=csv - ... --preview\={"unstable":true,"staging":true,"omni":"all"} Omni CASE Join Skips Unselected Error Branches @@ -12352,4 +12217,3 @@ Omni CASE Join Skips Unselected Error Branches ... ${query} ... n\n1 ... \-o\=csv - ... --preview\={"unstable":true,"staging":true,"omni":"all"} diff --git a/test/robot/functional/stackql_sessions_postgres.robot b/test/robot/functional/stackql_sessions_postgres.robot index 080eb0ccb..8ecb4d43e 100644 --- a/test/robot/functional/stackql_sessions_postgres.robot +++ b/test/robot/functional/stackql_sessions_postgres.robot @@ -30,12 +30,14 @@ SQLAlchemy Session Postgres PID Function ... stdout=${CURDIR}/tmp/SQLAlchemy-Session-Postgres-PID-Function.tmp [Teardown] NONE -SQLAlchemy Session Postgres Intel Views Exist - Log This test expects exactly 4 results per query in the sequence +SQLAlchemy Session Postgres Provider And Derived Reads + [Documentation] Provider reads and nested SQL remain available without legacy acquisition-backed intel views. + ${queries} = Evaluate + ... ["select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'testing-project' and zone = 'australia-southeast1-a' order by name desc", "select kind, name from (select kind, name from google.compute.acceleratorTypes where project = 'testing-project' and zone = 'australia-southeast1-a') t order by name desc", "select name, count(kind) as n from (select kind, name from google.compute.acceleratorTypes where project = 'testing-project' and zone = 'australia-southeast1-a') t group by name order by n desc"] Pass Execution If "${SQL_BACKEND}" != "postgres_tcp" This is a postgres only test Should SQLALchemy Raw Session Inline Have Length Greater Than Or Equal To ... ${POSTGRES_URL_UNENCRYPTED_CONN} - ... ${SELECT_ACCELERATOR_TYPES_DESC_SEQUENCE} + ... ${queries} ... 12 ... stdout=${CURDIR}/tmp/SQLAlchemy-Session-Postgres-Intel-Views-Exist.tmp [Teardown] NONE