From 9881c22c311a7aad562d427adea32e2ca01a2cab Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:55:09 +0000 Subject: [PATCH 1/4] Accept -o json on kernel search commands Search commands always print JSON, but they rejected the -o flag that other commands accept, so scripts piping through jq got an error on stderr instead of JSON. Register the shared output flag on search, get, providers, and contents, and validate it before any request is sent. --- cmd/search.go | 8 ++++++++ cmd/search_test.go | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/cmd/search.go b/cmd/search.go index 6184282b..3539afd0 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -69,6 +69,10 @@ func newSearchCommand() *cobra.Command { contents.Flags().String("content-browser-mode", "", "Browser retrieval mode: curl or render") contents.MarkFlagsMutuallyExclusive("result-ids", "limit") contents.MarkFlagsOneRequired("result-ids", "limit") + addJSONOutputFlag(cmd) + addJSONOutputFlag(get) + addJSONOutputFlag(providers) + addJSONOutputFlag(contents) cmd.AddCommand(get, providers, contents) return cmd } @@ -211,6 +215,10 @@ func validateSearchQuery(query string) error { } func executeSearchRequest(cmd *cobra.Command, method, path string, body json.RawMessage) error { + output, _ := cmd.Flags().GetString("output") + if err := validateJSONOutput(output); err != nil { + return err + } client := getKernelClient(cmd) var response json.RawMessage var opts []option.RequestOption diff --git a/cmd/search_test.go b/cmd/search_test.go index c6dc7dd9..c9aae4f1 100644 --- a/cmd/search_test.go +++ b/cmd/search_test.go @@ -74,6 +74,8 @@ func TestSearchReadCommands(t *testing.T) { {[]string{"get", "srch_test"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`}, {[]string{"providers"}, "/search/providers", "", `[]`}, {[]string{"providers", "--slug", "exa&other=1"}, "/search/providers", "slug=exa%26other%3D1", `[]`}, + {[]string{"get", "srch_test", "-o", "json"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`}, + {[]string{"providers", "-o", "json"}, "/search/providers", "", `[]`}, } { t.Run(strings.Join(tc.args, " "), func(t *testing.T) { stdout, err := executeSearchCommand(t, func(w http.ResponseWriter, r *http.Request) { @@ -99,6 +101,7 @@ func TestSearchContents(t *testing.T) { want string }{ {"limit", []string{"--limit", "3"}, `{"limit":3}`}, + {"limit with json output", []string{"--limit", "3", "-o", "json"}, `{"limit":3}`}, {"result ids", []string{"--result-ids", "res_2,res_1", "--timeout-ms", "30000"}, `{"result_ids":["res_2","res_1"],"timeout_ms":30000}`}, {"content options", []string{"--limit", "1", "--content-source", "browser", "--content-format", "text", "--content-max-chars", "500", "--content-max-age-hours", "0", "--content-timeout-ms", "5000", "--content-browser-id", "br_1", "--content-browser-mode", "render"}, `{"limit":1,"content":{"source":"browser","format":"text","max_chars":500,"max_age_hours":0,"timeout_ms":5000,"browser":{"browser_id":"br_1","mode":"render"}}}`}, @@ -127,6 +130,7 @@ func TestSearchInvalidInput(t *testing.T) { for _, args := range [][]string{ {}, {" "}, {strings.Repeat("x", 2049)}, {"a", "b"}, {"test", "--provider", ""}, {"test", "--max-results", "0"}, {"test", "--max-results", "101"}, + {"test", "-o", "yaml"}, {"get", "srch_test", "-o", "yaml"}, {"contents", "srch_test", "--limit", "1", "-o", "yaml"}, {"--request", "null"}, {"--request", "[]"}, {"--request", "{"}, {"--request", `{}`}, {"--request", `{"query":1}`}, {"test", "--request", `{"query":"test"}`}, {"--request", `{}`, "--provider", "exa"}, {"--request", `{}`, "--max-results", "5"}, {"--request-file", "request.json"}, From 0187fdababae79e632116bcf58a594a873727974 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:02:34 +0000 Subject: [PATCH 2/4] Document the content max chars range in search help --- README.md | 2 +- cmd/search.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bf3edc5..f8fa9cdb 100644 --- a/README.md +++ b/README.md @@ -1481,7 +1481,7 @@ Automated authentication for web services. The `run` command orchestrates the fu - `--show-content` - Print the extracted content text for each result (implies `--content`) - `--content-source ` - Retrieval source: `auto`, `provider`, or `browser` - `--content-format ` - Extracted content format: `markdown` or `text` - - `--content-max-chars ` - Per-result Unicode character limit after extraction + - `--content-max-chars ` - Per-result Unicode character limit after extraction (100–100000; default 10000) - `--content-max-age-hours ` - Maximum acceptable age of cached page content; `0` forces a live fetch - `--content-timeout-ms ` - Per-result retrieval deadline - `--content-browser-id ` - Retrieve through an existing browser session (requires `--content-source browser`) diff --git a/cmd/search.go b/cmd/search.go index 3539afd0..1e6b15d2 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -62,7 +62,7 @@ func newSearchCommand() *cobra.Command { contents.Flags().Int("timeout-ms", 0, "Overall deadline across all selected results in milliseconds (1000–120000; default 60000)") contents.Flags().String("content-source", "", "Content source: auto, provider, or browser (default auto)") contents.Flags().String("content-format", "", "Content format: markdown or text") - contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction") + contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction (100–100000; default 10000)") contents.Flags().Int("content-max-age-hours", 0, "For source auto, maximum age of retained provider content; 0 fetches every result through a browser") contents.Flags().Int("content-timeout-ms", 0, "Per-result deadline in milliseconds, including capacity acquisition, retrieval, and extraction") contents.Flags().String("content-browser-id", "", "Existing browser session to retrieve through (requires source auto or browser)") From 2b15205ace8a4634fccb69c2ae75f91ea7f274aa Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:21:59 +0000 Subject: [PATCH 3/4] Accept a JSON request on search contents and clamp max chars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add --request to `search contents`, taking the full request body as a JSON object and sending it unchanged. It cannot be combined with the other contents flags. - Clamp --content-max-chars to the API's 100–100000 range instead of passing out-of-range values through to a server error. Negative values still error. --- README.md | 2 +- cmd/search.go | 26 +++++++++++++++++++++++--- cmd/search_test.go | 5 +++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f8fa9cdb..c021bd09 100644 --- a/README.md +++ b/README.md @@ -1481,7 +1481,7 @@ Automated authentication for web services. The `run` command orchestrates the fu - `--show-content` - Print the extracted content text for each result (implies `--content`) - `--content-source ` - Retrieval source: `auto`, `provider`, or `browser` - `--content-format ` - Extracted content format: `markdown` or `text` - - `--content-max-chars ` - Per-result Unicode character limit after extraction (100–100000; default 10000) + - `--content-max-chars ` - Per-result Unicode character limit after extraction (clamped to 100–100000; default 10000) - `--content-max-age-hours ` - Maximum acceptable age of cached page content; `0` forces a live fetch - `--content-timeout-ms ` - Per-result retrieval deadline - `--content-browser-id ` - Retrieve through an existing browser session (requires `--content-source browser`) diff --git a/cmd/search.go b/cmd/search.go index 1e6b15d2..c3f33424 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -62,13 +62,14 @@ func newSearchCommand() *cobra.Command { contents.Flags().Int("timeout-ms", 0, "Overall deadline across all selected results in milliseconds (1000–120000; default 60000)") contents.Flags().String("content-source", "", "Content source: auto, provider, or browser (default auto)") contents.Flags().String("content-format", "", "Content format: markdown or text") - contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction (100–100000; default 10000)") + contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction (clamped to 100–100000; default 10000)") contents.Flags().Int("content-max-age-hours", 0, "For source auto, maximum age of retained provider content; 0 fetches every result through a browser") contents.Flags().Int("content-timeout-ms", 0, "Per-result deadline in milliseconds, including capacity acquisition, retrieval, and extraction") contents.Flags().String("content-browser-id", "", "Existing browser session to retrieve through (requires source auto or browser)") contents.Flags().String("content-browser-mode", "", "Browser retrieval mode: curl or render") + contents.Flags().String("request", "", "Complete contents request as a JSON object; cannot be combined with other contents flags") contents.MarkFlagsMutuallyExclusive("result-ids", "limit") - contents.MarkFlagsOneRequired("result-ids", "limit") + contents.MarkFlagsOneRequired("result-ids", "limit", "request") addJSONOutputFlag(cmd) addJSONOutputFlag(get) addJSONOutputFlag(providers) @@ -82,6 +83,19 @@ func runSearchContents(cmd *cobra.Command, args []string) error { return fmt.Errorf("search ID must not be empty") } flags := cmd.Flags() + if flags.Changed("request") { + for _, name := range []string{"result-ids", "limit", "timeout-ms", "content-source", "content-format", "content-max-chars", "content-max-age-hours", "content-timeout-ms", "content-browser-id", "content-browser-mode"} { + if flags.Changed(name) { + return fmt.Errorf("--request cannot be combined with --%s", name) + } + } + input, _ := flags.GetString("request") + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(input), &fields); err != nil || fields == nil { + return fmt.Errorf("contents request must be a JSON object") + } + return executeSearchRequest(cmd, http.MethodPost, "search/"+url.PathEscape(args[0])+"/contents", json.RawMessage(input)) + } var request kernel.FetchRequestParam if flags.Changed("result-ids") { ids, _ := flags.GetStringSlice("result-ids") @@ -128,7 +142,6 @@ func runSearchContents(cmd *cobra.Command, args []string) error { return err } for name, target := range map[string]*param.Opt[int64]{ - "content-max-chars": &content.MaxChars, "content-max-age-hours": &content.MaxAgeHours, "content-timeout-ms": &content.TimeoutMs, } { @@ -140,6 +153,13 @@ func runSearchContents(cmd *cobra.Command, args []string) error { *target = kernel.Int(int64(value)) } } + if flags.Changed("content-max-chars") { + value, _ := flags.GetInt("content-max-chars") + if value < 0 { + return fmt.Errorf("--content-max-chars must not be negative") + } + content.MaxChars = kernel.Int(int64(min(max(value, 100), 100000))) + } if flags.Changed("content-browser-id") { browserID, _ := flags.GetString("content-browser-id") if strings.TrimSpace(browserID) == "" { diff --git a/cmd/search_test.go b/cmd/search_test.go index c9aae4f1..351978c9 100644 --- a/cmd/search_test.go +++ b/cmd/search_test.go @@ -102,6 +102,9 @@ func TestSearchContents(t *testing.T) { }{ {"limit", []string{"--limit", "3"}, `{"limit":3}`}, {"limit with json output", []string{"--limit", "3", "-o", "json"}, `{"limit":3}`}, + {"request body", []string{"--request", `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`}, `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`}, + {"max chars above range is clamped", []string{"--limit", "1", "--content-max-chars", "120000"}, `{"limit":1,"content":{"max_chars":100000}}`}, + {"max chars below range is clamped", []string{"--limit", "1", "--content-max-chars", "50"}, `{"limit":1,"content":{"max_chars":100}}`}, {"result ids", []string{"--result-ids", "res_2,res_1", "--timeout-ms", "30000"}, `{"result_ids":["res_2","res_1"],"timeout_ms":30000}`}, {"content options", []string{"--limit", "1", "--content-source", "browser", "--content-format", "text", "--content-max-chars", "500", "--content-max-age-hours", "0", "--content-timeout-ms", "5000", "--content-browser-id", "br_1", "--content-browser-mode", "render"}, `{"limit":1,"content":{"source":"browser","format":"text","max_chars":500,"max_age_hours":0,"timeout_ms":5000,"browser":{"browser_id":"br_1","mode":"render"}}}`}, @@ -143,6 +146,8 @@ func TestSearchInvalidInput(t *testing.T) { {"contents", "srch_test", "--limit", "1", "--content-format", "html"}, {"contents", "srch_test", "--limit", "1", "--content-browser-mode", "headless"}, {"contents", "srch_test", "--limit", "1", "--content-max-chars", "-1"}, + {"contents", "srch_test", "--request", "[]"}, + {"contents", "srch_test", "--request", `{"limit":1}`, "--limit", "1"}, {"contents", "srch_test", "--limit", "1", "--content-browser-id", " "}, {"contents", "srch_test", "--result-ids", "res_1,"}, } { From 96ae0f9a0960e65fcd847d247fdb557e1ec58fed Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:13:52 +0000 Subject: [PATCH 4/4] Address review on search output and contents request - Register -o/--output once on the search command as a persistent flag. - Let cobra enforce that --request cannot be combined with the other contents flags, so new flags are covered automatically. - Share JSON object parsing between search and contents --request. - Warn on stderr when --content-max-chars is clamped. - Move the clamp note to the search contents section of the README. --- README.md | 6 +++-- cmd/search.go | 56 ++++++++++++++++++++++++++++------------------ cmd/search_test.go | 1 + 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c0bcb7b0..82d8fa20 100644 --- a/README.md +++ b/README.md @@ -1522,7 +1522,7 @@ Automated authentication for web services. The `run` command orchestrates the fu - `--show-content` - Print the extracted content text for each result (implies `--content`) - `--content-source ` - Retrieval source: `auto`, `provider`, or `browser` - `--content-format ` - Extracted content format: `markdown` or `text` - - `--content-max-chars ` - Per-result Unicode character limit after extraction (clamped to 100–100000; default 10000) + - `--content-max-chars ` - Per-result Unicode character limit after extraction - `--content-max-age-hours ` - Maximum acceptable age of cached page content; `0` forces a live fetch - `--content-timeout-ms ` - Per-result retrieval deadline - `--content-browser-id ` - Retrieve through an existing browser session (requires `--content-source browser`) @@ -1544,11 +1544,13 @@ Automated authentication for web services. The `run` command orchestrates the fu - `--timeout-ms ` - Overall deadline across all selected results - `--content-source ` - `auto` (default), `provider`, or `browser` - `--content-format ` - `markdown` or `text` - - `--content-max-chars ` - Per-result character limit after extraction + - `--content-max-chars ` - Per-result character limit after extraction, clamped to 100–100000 (default 10000) - `--content-max-age-hours ` - For `auto`, maximum age of retained provider content; `0` always uses a browser - `--content-timeout-ms ` - Per-result deadline - `--content-browser-id ` - Retrieve through an existing browser session - `--content-browser-mode ` - `curl` or `render` + - `--request ` - Complete request body as a JSON object (`result_ids` or `limit`, plus optional `timeout_ms` and `content`); cannot be combined with the other contents flags + - `--output json`, `-o json` - Output raw JSON object Searches are retained for 24 hours. Omitting the strategy flags lets Kernel pick an eligible provider; portable filters a provider cannot honor are approximated or diff --git a/cmd/search.go b/cmd/search.go index c3f33424..7556d66c 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -14,6 +14,7 @@ import ( "github.com/kernel/kernel-go-sdk/option" "github.com/kernel/kernel-go-sdk/packages/param" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) func newSearchCommand() *cobra.Command { @@ -27,6 +28,7 @@ func newSearchCommand() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: runSearch, } + cmd.PersistentFlags().StringP("output", "o", "", util.JSONOutputFlagDescription) cmd.Flags().String("provider", "", "Pin a provider slug (default: automatic routing)") cmd.Flags().Int("max-results", 10, "Requested result count (1–100; subject to provider cap)") cmd.Flags().String("request", "", "Complete Search API request as a JSON object; cannot be combined with query flags") @@ -50,10 +52,10 @@ func newSearchCommand() *cobra.Command { contents := &cobra.Command{ Use: "contents ", Short: "Fetch content for selected results of a retained search as JSON", - Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit; the latter fetches the top results. Content defaults to source auto. " + + Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit, or a complete request with --request; the --limit form fetches the top results. Content defaults to source auto. " + "Responses preserve --result-ids order and include one outcome per selected result, including timeout entries for work unfinished at the overall deadline. " + "Browser retrievals run sequentially in result order and are billed like any other browser.", - Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render", + Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render\n kernel search contents srch_123 --request '{\"result_ids\":[\"res_1\"],\"content\":{\"source\":\"browser\"}}'", Args: cobra.ExactArgs(1), RunE: runSearchContents, } @@ -70,10 +72,11 @@ func newSearchCommand() *cobra.Command { contents.Flags().String("request", "", "Complete contents request as a JSON object; cannot be combined with other contents flags") contents.MarkFlagsMutuallyExclusive("result-ids", "limit") contents.MarkFlagsOneRequired("result-ids", "limit", "request") - addJSONOutputFlag(cmd) - addJSONOutputFlag(get) - addJSONOutputFlag(providers) - addJSONOutputFlag(contents) + contents.Flags().VisitAll(func(f *pflag.Flag) { + if f.Name != "request" { + contents.MarkFlagsMutuallyExclusive("request", f.Name) + } + }) cmd.AddCommand(get, providers, contents) return cmd } @@ -84,17 +87,12 @@ func runSearchContents(cmd *cobra.Command, args []string) error { } flags := cmd.Flags() if flags.Changed("request") { - for _, name := range []string{"result-ids", "limit", "timeout-ms", "content-source", "content-format", "content-max-chars", "content-max-age-hours", "content-timeout-ms", "content-browser-id", "content-browser-mode"} { - if flags.Changed(name) { - return fmt.Errorf("--request cannot be combined with --%s", name) - } - } input, _ := flags.GetString("request") - var fields map[string]json.RawMessage - if err := json.Unmarshal([]byte(input), &fields); err != nil || fields == nil { - return fmt.Errorf("contents request must be a JSON object") + body, _, err := parseJSONObject(input, "contents request") + if err != nil { + return err } - return executeSearchRequest(cmd, http.MethodPost, "search/"+url.PathEscape(args[0])+"/contents", json.RawMessage(input)) + return executeSearchRequest(cmd, http.MethodPost, "search/"+url.PathEscape(args[0])+"/contents", body) } var request kernel.FetchRequestParam if flags.Changed("result-ids") { @@ -158,7 +156,11 @@ func runSearchContents(cmd *cobra.Command, args []string) error { if value < 0 { return fmt.Errorf("--content-max-chars must not be negative") } - content.MaxChars = kernel.Int(int64(min(max(value, 100), 100000))) + clamped := min(max(value, 100), 100000) + if clamped != value { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: --content-max-chars %d clamped to %d\n", value, clamped) + } + content.MaxChars = kernel.Int(int64(clamped)) } if flags.Changed("content-browser-id") { browserID, _ := flags.GetString("content-browser-id") @@ -182,11 +184,10 @@ func runSearch(cmd *cobra.Command, args []string) error { } input, _ := cmd.Flags().GetString("request") var fields map[string]json.RawMessage - if err := json.Unmarshal([]byte(input), &fields); err != nil { - return fmt.Errorf("search request must be a JSON object: %w", err) - } - if fields == nil { - return fmt.Errorf("search request must be a JSON object") + var err error + body, fields, err = parseJSONObject(input, "search request") + if err != nil { + return err } var query string if err := json.Unmarshal(fields["query"], &query); err != nil { @@ -195,7 +196,6 @@ func runSearch(cmd *cobra.Command, args []string) error { if err := validateSearchQuery(query); err != nil { return err } - body = json.RawMessage(input) } else { if len(args) != 1 { return fmt.Errorf("provide a query or --request") @@ -227,6 +227,18 @@ func runSearch(cmd *cobra.Command, args []string) error { return executeSearchRequest(cmd, http.MethodPost, "search", body) } +// parseJSONObject returns the raw input and its top-level fields when input is a JSON object. +func parseJSONObject(input, label string) (json.RawMessage, map[string]json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(input), &fields); err != nil { + return nil, nil, fmt.Errorf("%s must be a JSON object: %w", label, err) + } + if fields == nil { + return nil, nil, fmt.Errorf("%s must be a JSON object", label) + } + return json.RawMessage(input), fields, nil +} + func validateSearchQuery(query string) error { if strings.TrimSpace(query) == "" || utf8.RuneCountInString(query) > 2048 { return fmt.Errorf("query must contain 1–2048 characters and not be blank") diff --git a/cmd/search_test.go b/cmd/search_test.go index 351978c9..5689c738 100644 --- a/cmd/search_test.go +++ b/cmd/search_test.go @@ -148,6 +148,7 @@ func TestSearchInvalidInput(t *testing.T) { {"contents", "srch_test", "--limit", "1", "--content-max-chars", "-1"}, {"contents", "srch_test", "--request", "[]"}, {"contents", "srch_test", "--request", `{"limit":1}`, "--limit", "1"}, + {"contents", "srch_test", "--request", `{"limit":1}`, "--content-source", "browser"}, {"contents", "srch_test", "--limit", "1", "--content-browser-id", " "}, {"contents", "srch_test", "--result-ids", "res_1,"}, } {