Skip to content

Accept -o json on kernel search commands and extend search contents #297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chruffins merged 5 commits into from
Oct 26, 2026
+64 −12
Merged

Accept -o json on kernel search commands and extend search contents #297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9881c22
Accept -o json on kernel search commands
chruffins Oct 13, 2026
0187fda
Document the content max chars range in search help
chruffins Oct 13, 2026
2b15205
Accept a JSON request on search contents and clamp max chars
chruffins Oct 26, 2026
fd0c6df
Merge branch 'main' into hypeship/search-json-output-flag
chruffins Oct 26, 2026
96ae0f9
Address review on search output and contents request
chruffins Oct 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1544,11 +1544,13 @@ Automated authentication for web services. The `run` command orchestrates the fu
- `--timeout-ms <ms>` - Overall deadline across all selected results
- `--content-source <source>` - `auto` (default), `provider`, or `browser`
- `--content-format <format>` - `markdown` or `text`
- `--content-max-chars <n>` - Per-result character limit after extraction
- `--content-max-chars <n>` - Per-result character limit after extraction, clamped to 100–100000 (default 10000)
- `--content-max-age-hours <n>` - For `auto`, maximum age of retained provider content; `0` always uses a browser
- `--content-timeout-ms <ms>` - Per-result deadline
- `--content-browser-id <id>` - Retrieve through an existing browser session
- `--content-browser-mode <mode>` - `curl` or `render`
- `--request <json>` - Complete request body as a JSON object (`result_ids` or `limit`, plus optional `timeout_ms` and `content`); cannot be combined with the other contents flags
- `--output json`, `-o json` - Output raw JSON object

Searches are retained for 24 hours. Omitting the strategy flags lets Kernel pick an
eligible provider; portable filters a provider cannot honor are approximated or
Expand Down
62 changes: 51 additions & 11 deletions cmd/search.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/kernel/kernel-go-sdk/option"
"github.com/kernel/kernel-go-sdk/packages/param"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)

func newSearchCommand() *cobra.Command {
Expand All @@ -27,6 +28,7 @@ func newSearchCommand() *cobra.Command {
Args: cobra.MaximumNArgs(1),
RunE: runSearch,
}
cmd.PersistentFlags().StringP("output", "o", "", util.JSONOutputFlagDescription)
cmd.Flags().String("provider", "", "Pin a provider slug (default: automatic routing)")
cmd.Flags().Int("max-results", 10, "Requested result count (1–100; subject to provider cap)")
cmd.Flags().String("request", "", "Complete Search API request as a JSON object; cannot be combined with query flags")
Expand All @@ -50,10 +52,10 @@ func newSearchCommand() *cobra.Command {
contents := &cobra.Command{
Use: "contents <id>",
Short: "Fetch content for selected results of a retained search as JSON",
Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit; the latter fetches the top results. Content defaults to source auto. " +
Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit, or a complete request with --request; the --limit form fetches the top results. Content defaults to source auto. " +
"Responses preserve --result-ids order and include one outcome per selected result, including timeout entries for work unfinished at the overall deadline. " +
"Browser retrievals run sequentially in result order and are billed like any other browser.",
Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render",
Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render\n kernel search contents srch_123 --request '{\"result_ids\":[\"res_1\"],\"content\":{\"source\":\"browser\"}}'",
Args: cobra.ExactArgs(1),
RunE: runSearchContents,
}
Expand All @@ -62,13 +64,19 @@ func newSearchCommand() *cobra.Command {
contents.Flags().Int("timeout-ms", 0, "Overall deadline across all selected results in milliseconds (1000–120000; default 60000)")
contents.Flags().String("content-source", "", "Content source: auto, provider, or browser (default auto)")
contents.Flags().String("content-format", "", "Content format: markdown or text")
contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction")
contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction (clamped to 100–100000; default 10000)")
contents.Flags().Int("content-max-age-hours", 0, "For source auto, maximum age of retained provider content; 0 fetches every result through a browser")
contents.Flags().Int("content-timeout-ms", 0, "Per-result deadline in milliseconds, including capacity acquisition, retrieval, and extraction")
contents.Flags().String("content-browser-id", "", "Existing browser session to retrieve through (requires source auto or browser)")
contents.Flags().String("content-browser-mode", "", "Browser retrieval mode: curl or render")
contents.Flags().String("request", "", "Complete contents request as a JSON object; cannot be combined with other contents flags")
contents.MarkFlagsMutuallyExclusive("result-ids", "limit")
contents.MarkFlagsOneRequired("result-ids", "limit")
contents.MarkFlagsOneRequired("result-ids", "limit", "request")
contents.Flags().VisitAll(func(f *pflag.Flag) {
if f.Name != "request" {
contents.MarkFlagsMutuallyExclusive("request", f.Name)
}
})
cmd.AddCommand(get, providers, contents)
return cmd
}
Expand All @@ -78,6 +86,14 @@ func runSearchContents(cmd *cobra.Command, args []string) error {
return fmt.Errorf("search ID must not be empty")
}
flags := cmd.Flags()
if flags.Changed("request") {
input, _ := flags.GetString("request")
body, _, err := parseJSONObject(input, "contents request")
if err != nil {
return err
}
return executeSearchRequest(cmd, http.MethodPost, "search/"+url.PathEscape(args[0])+"/contents", body)
}
var request kernel.FetchRequestParam
if flags.Changed("result-ids") {
ids, _ := flags.GetStringSlice("result-ids")
Expand Down Expand Up @@ -124,7 +140,6 @@ func runSearchContents(cmd *cobra.Command, args []string) error {
return err
}
for name, target := range map[string]*param.Opt[int64]{
"content-max-chars": &content.MaxChars,
"content-max-age-hours": &content.MaxAgeHours,
"content-timeout-ms": &content.TimeoutMs,
} {
Expand All @@ -136,6 +151,17 @@ func runSearchContents(cmd *cobra.Command, args []string) error {
*target = kernel.Int(int64(value))
}
}
if flags.Changed("content-max-chars") {
value, _ := flags.GetInt("content-max-chars")
if value < 0 {
return fmt.Errorf("--content-max-chars must not be negative")
}
clamped := min(max(value, 100), 100000)
if clamped != value {
fmt.Fprintf(cmd.ErrOrStderr(), "warning: --content-max-chars %d clamped to %d\n", value, clamped)
}
content.MaxChars = kernel.Int(int64(clamped))
}
if flags.Changed("content-browser-id") {
browserID, _ := flags.GetString("content-browser-id")
if strings.TrimSpace(browserID) == "" {
Expand All @@ -158,11 +184,10 @@ func runSearch(cmd *cobra.Command, args []string) error {
}
input, _ := cmd.Flags().GetString("request")
var fields map[string]json.RawMessage
if err := json.Unmarshal([]byte(input), &fields); err != nil {
return fmt.Errorf("search request must be a JSON object: %w", err)
}
if fields == nil {
return fmt.Errorf("search request must be a JSON object")
var err error
body, fields, err = parseJSONObject(input, "search request")
if err != nil {
return err
}
var query string
if err := json.Unmarshal(fields["query"], &query); err != nil {
Expand All @@ -171,7 +196,6 @@ func runSearch(cmd *cobra.Command, args []string) error {
if err := validateSearchQuery(query); err != nil {
return err
}
body = json.RawMessage(input)
} else {
if len(args) != 1 {
return fmt.Errorf("provide a query or --request")
Expand Down Expand Up @@ -203,6 +227,18 @@ func runSearch(cmd *cobra.Command, args []string) error {
return executeSearchRequest(cmd, http.MethodPost, "search", body)
}

// parseJSONObject returns the raw input and its top-level fields when input is a JSON object.
func parseJSONObject(input, label string) (json.RawMessage, map[string]json.RawMessage, error) {
var fields map[string]json.RawMessage
if err := json.Unmarshal([]byte(input), &fields); err != nil {
return nil, nil, fmt.Errorf("%s must be a JSON object: %w", label, err)
}
if fields == nil {
return nil, nil, fmt.Errorf("%s must be a JSON object", label)
}
return json.RawMessage(input), fields, nil
}

func validateSearchQuery(query string) error {
if strings.TrimSpace(query) == "" || utf8.RuneCountInString(query) > 2048 {
return fmt.Errorf("query must contain 1–2048 characters and not be blank")
Expand All @@ -211,6 +247,10 @@ func validateSearchQuery(query string) error {
}

func executeSearchRequest(cmd *cobra.Command, method, path string, body json.RawMessage) error {
output, _ := cmd.Flags().GetString("output")
if err := validateJSONOutput(output); err != nil {
return err
}
client := getKernelClient(cmd)
var response json.RawMessage
var opts []option.RequestOption
Expand Down
10 changes: 10 additions & 0 deletions cmd/search_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ func TestSearchReadCommands(t *testing.T) {
{[]string{"get", "srch_test"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`},
{[]string{"providers"}, "/search/providers", "", `[]`},
{[]string{"providers", "--slug", "exa&other=1"}, "/search/providers", "slug=exa%26other%3D1", `[]`},
{[]string{"get", "srch_test", "-o", "json"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`},
{[]string{"providers", "-o", "json"}, "/search/providers", "", `[]`},
} {
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
stdout, err := executeSearchCommand(t, func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -99,6 +101,10 @@ func TestSearchContents(t *testing.T) {
want string
}{
{"limit", []string{"--limit", "3"}, `{"limit":3}`},
{"limit with json output", []string{"--limit", "3", "-o", "json"}, `{"limit":3}`},
{"request body", []string{"--request", `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`}, `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`},
{"max chars above range is clamped", []string{"--limit", "1", "--content-max-chars", "120000"}, `{"limit":1,"content":{"max_chars":100000}}`},
{"max chars below range is clamped", []string{"--limit", "1", "--content-max-chars", "50"}, `{"limit":1,"content":{"max_chars":100}}`},
{"result ids", []string{"--result-ids", "res_2,res_1", "--timeout-ms", "30000"}, `{"result_ids":["res_2","res_1"],"timeout_ms":30000}`},
{"content options", []string{"--limit", "1", "--content-source", "browser", "--content-format", "text", "--content-max-chars", "500", "--content-max-age-hours", "0", "--content-timeout-ms", "5000", "--content-browser-id", "br_1", "--content-browser-mode", "render"},
`{"limit":1,"content":{"source":"browser","format":"text","max_chars":500,"max_age_hours":0,"timeout_ms":5000,"browser":{"browser_id":"br_1","mode":"render"}}}`},
Expand Down Expand Up @@ -127,6 +133,7 @@ func TestSearchInvalidInput(t *testing.T) {
for _, args := range [][]string{
{}, {" "}, {strings.Repeat("x", 2049)}, {"a", "b"}, {"test", "--provider", ""},
{"test", "--max-results", "0"}, {"test", "--max-results", "101"},
{"test", "-o", "yaml"}, {"get", "srch_test", "-o", "yaml"}, {"contents", "srch_test", "--limit", "1", "-o", "yaml"},
{"--request", "null"}, {"--request", "[]"}, {"--request", "{"}, {"--request", `{}`}, {"--request", `{"query":1}`},
{"test", "--request", `{"query":"test"}`}, {"--request", `{}`, "--provider", "exa"},
{"--request", `{}`, "--max-results", "5"}, {"--request-file", "request.json"},
Expand All @@ -139,6 +146,9 @@ func TestSearchInvalidInput(t *testing.T) {
{"contents", "srch_test", "--limit", "1", "--content-format", "html"},
{"contents", "srch_test", "--limit", "1", "--content-browser-mode", "headless"},
{"contents", "srch_test", "--limit", "1", "--content-max-chars", "-1"},
{"contents", "srch_test", "--request", "[]"},
{"contents", "srch_test", "--request", `{"limit":1}`, "--limit", "1"},
{"contents", "srch_test", "--request", `{"limit":1}`, "--content-source", "browser"},
{"contents", "srch_test", "--limit", "1", "--content-browser-id", " "},
{"contents", "srch_test", "--result-ids", "res_1,"},
} {
Expand Down
Loading