diff --git a/README.md b/README.md index 2a407744..82d8fa20 100644 --- a/README.md +++ b/README.md @@ -1544,11 +1544,13 @@ Automated authentication for web services. The `run` command orchestrates the fu - `--timeout-ms ` - Overall deadline across all selected results - `--content-source ` - `auto` (default), `provider`, or `browser` - `--content-format ` - `markdown` or `text` - - `--content-max-chars ` - Per-result character limit after extraction + - `--content-max-chars ` - Per-result character limit after extraction, clamped to 100–100000 (default 10000) - `--content-max-age-hours ` - For `auto`, maximum age of retained provider content; `0` always uses a browser - `--content-timeout-ms ` - Per-result deadline - `--content-browser-id ` - Retrieve through an existing browser session - `--content-browser-mode ` - `curl` or `render` + - `--request ` - Complete request body as a JSON object (`result_ids` or `limit`, plus optional `timeout_ms` and `content`); cannot be combined with the other contents flags + - `--output json`, `-o json` - Output raw JSON object Searches are retained for 24 hours. Omitting the strategy flags lets Kernel pick an eligible provider; portable filters a provider cannot honor are approximated or diff --git a/cmd/search.go b/cmd/search.go index 6184282b..7556d66c 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -14,6 +14,7 @@ import ( "github.com/kernel/kernel-go-sdk/option" "github.com/kernel/kernel-go-sdk/packages/param" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) func newSearchCommand() *cobra.Command { @@ -27,6 +28,7 @@ func newSearchCommand() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: runSearch, } + cmd.PersistentFlags().StringP("output", "o", "", util.JSONOutputFlagDescription) cmd.Flags().String("provider", "", "Pin a provider slug (default: automatic routing)") cmd.Flags().Int("max-results", 10, "Requested result count (1–100; subject to provider cap)") cmd.Flags().String("request", "", "Complete Search API request as a JSON object; cannot be combined with query flags") @@ -50,10 +52,10 @@ func newSearchCommand() *cobra.Command { contents := &cobra.Command{ Use: "contents ", Short: "Fetch content for selected results of a retained search as JSON", - Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit; the latter fetches the top results. Content defaults to source auto. " + + Long: "Retrieves selected results from a retained search. Provide exactly one of --result-ids or --limit, or a complete request with --request; the --limit form fetches the top results. Content defaults to source auto. " + "Responses preserve --result-ids order and include one outcome per selected result, including timeout entries for work unfinished at the overall deadline. " + "Browser retrievals run sequentially in result order and are billed like any other browser.", - Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render", + Example: " kernel search contents srch_123 --limit 3\n kernel search contents srch_123 --result-ids res_1,res_2 --content-source browser --content-browser-mode render\n kernel search contents srch_123 --request '{\"result_ids\":[\"res_1\"],\"content\":{\"source\":\"browser\"}}'", Args: cobra.ExactArgs(1), RunE: runSearchContents, } @@ -62,13 +64,19 @@ func newSearchCommand() *cobra.Command { contents.Flags().Int("timeout-ms", 0, "Overall deadline across all selected results in milliseconds (1000–120000; default 60000)") contents.Flags().String("content-source", "", "Content source: auto, provider, or browser (default auto)") contents.Flags().String("content-format", "", "Content format: markdown or text") - contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction") + contents.Flags().Int("content-max-chars", 0, "Per-result Unicode character limit after extraction (clamped to 100–100000; default 10000)") contents.Flags().Int("content-max-age-hours", 0, "For source auto, maximum age of retained provider content; 0 fetches every result through a browser") contents.Flags().Int("content-timeout-ms", 0, "Per-result deadline in milliseconds, including capacity acquisition, retrieval, and extraction") contents.Flags().String("content-browser-id", "", "Existing browser session to retrieve through (requires source auto or browser)") contents.Flags().String("content-browser-mode", "", "Browser retrieval mode: curl or render") + contents.Flags().String("request", "", "Complete contents request as a JSON object; cannot be combined with other contents flags") contents.MarkFlagsMutuallyExclusive("result-ids", "limit") - contents.MarkFlagsOneRequired("result-ids", "limit") + contents.MarkFlagsOneRequired("result-ids", "limit", "request") + contents.Flags().VisitAll(func(f *pflag.Flag) { + if f.Name != "request" { + contents.MarkFlagsMutuallyExclusive("request", f.Name) + } + }) cmd.AddCommand(get, providers, contents) return cmd } @@ -78,6 +86,14 @@ func runSearchContents(cmd *cobra.Command, args []string) error { return fmt.Errorf("search ID must not be empty") } flags := cmd.Flags() + if flags.Changed("request") { + input, _ := flags.GetString("request") + body, _, err := parseJSONObject(input, "contents request") + if err != nil { + return err + } + return executeSearchRequest(cmd, http.MethodPost, "search/"+url.PathEscape(args[0])+"/contents", body) + } var request kernel.FetchRequestParam if flags.Changed("result-ids") { ids, _ := flags.GetStringSlice("result-ids") @@ -124,7 +140,6 @@ func runSearchContents(cmd *cobra.Command, args []string) error { return err } for name, target := range map[string]*param.Opt[int64]{ - "content-max-chars": &content.MaxChars, "content-max-age-hours": &content.MaxAgeHours, "content-timeout-ms": &content.TimeoutMs, } { @@ -136,6 +151,17 @@ func runSearchContents(cmd *cobra.Command, args []string) error { *target = kernel.Int(int64(value)) } } + if flags.Changed("content-max-chars") { + value, _ := flags.GetInt("content-max-chars") + if value < 0 { + return fmt.Errorf("--content-max-chars must not be negative") + } + clamped := min(max(value, 100), 100000) + if clamped != value { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: --content-max-chars %d clamped to %d\n", value, clamped) + } + content.MaxChars = kernel.Int(int64(clamped)) + } if flags.Changed("content-browser-id") { browserID, _ := flags.GetString("content-browser-id") if strings.TrimSpace(browserID) == "" { @@ -158,11 +184,10 @@ func runSearch(cmd *cobra.Command, args []string) error { } input, _ := cmd.Flags().GetString("request") var fields map[string]json.RawMessage - if err := json.Unmarshal([]byte(input), &fields); err != nil { - return fmt.Errorf("search request must be a JSON object: %w", err) - } - if fields == nil { - return fmt.Errorf("search request must be a JSON object") + var err error + body, fields, err = parseJSONObject(input, "search request") + if err != nil { + return err } var query string if err := json.Unmarshal(fields["query"], &query); err != nil { @@ -171,7 +196,6 @@ func runSearch(cmd *cobra.Command, args []string) error { if err := validateSearchQuery(query); err != nil { return err } - body = json.RawMessage(input) } else { if len(args) != 1 { return fmt.Errorf("provide a query or --request") @@ -203,6 +227,18 @@ func runSearch(cmd *cobra.Command, args []string) error { return executeSearchRequest(cmd, http.MethodPost, "search", body) } +// parseJSONObject returns the raw input and its top-level fields when input is a JSON object. +func parseJSONObject(input, label string) (json.RawMessage, map[string]json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(input), &fields); err != nil { + return nil, nil, fmt.Errorf("%s must be a JSON object: %w", label, err) + } + if fields == nil { + return nil, nil, fmt.Errorf("%s must be a JSON object", label) + } + return json.RawMessage(input), fields, nil +} + func validateSearchQuery(query string) error { if strings.TrimSpace(query) == "" || utf8.RuneCountInString(query) > 2048 { return fmt.Errorf("query must contain 1–2048 characters and not be blank") @@ -211,6 +247,10 @@ func validateSearchQuery(query string) error { } func executeSearchRequest(cmd *cobra.Command, method, path string, body json.RawMessage) error { + output, _ := cmd.Flags().GetString("output") + if err := validateJSONOutput(output); err != nil { + return err + } client := getKernelClient(cmd) var response json.RawMessage var opts []option.RequestOption diff --git a/cmd/search_test.go b/cmd/search_test.go index c6dc7dd9..5689c738 100644 --- a/cmd/search_test.go +++ b/cmd/search_test.go @@ -74,6 +74,8 @@ func TestSearchReadCommands(t *testing.T) { {[]string{"get", "srch_test"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`}, {[]string{"providers"}, "/search/providers", "", `[]`}, {[]string{"providers", "--slug", "exa&other=1"}, "/search/providers", "slug=exa%26other%3D1", `[]`}, + {[]string{"get", "srch_test", "-o", "json"}, "/search/srch_test", "", `{"id":"srch_test","results":[]}`}, + {[]string{"providers", "-o", "json"}, "/search/providers", "", `[]`}, } { t.Run(strings.Join(tc.args, " "), func(t *testing.T) { stdout, err := executeSearchCommand(t, func(w http.ResponseWriter, r *http.Request) { @@ -99,6 +101,10 @@ func TestSearchContents(t *testing.T) { want string }{ {"limit", []string{"--limit", "3"}, `{"limit":3}`}, + {"limit with json output", []string{"--limit", "3", "-o", "json"}, `{"limit":3}`}, + {"request body", []string{"--request", `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`}, `{"result_ids":["res_1"],"content":{"source":"browser","max_chars":120000}}`}, + {"max chars above range is clamped", []string{"--limit", "1", "--content-max-chars", "120000"}, `{"limit":1,"content":{"max_chars":100000}}`}, + {"max chars below range is clamped", []string{"--limit", "1", "--content-max-chars", "50"}, `{"limit":1,"content":{"max_chars":100}}`}, {"result ids", []string{"--result-ids", "res_2,res_1", "--timeout-ms", "30000"}, `{"result_ids":["res_2","res_1"],"timeout_ms":30000}`}, {"content options", []string{"--limit", "1", "--content-source", "browser", "--content-format", "text", "--content-max-chars", "500", "--content-max-age-hours", "0", "--content-timeout-ms", "5000", "--content-browser-id", "br_1", "--content-browser-mode", "render"}, `{"limit":1,"content":{"source":"browser","format":"text","max_chars":500,"max_age_hours":0,"timeout_ms":5000,"browser":{"browser_id":"br_1","mode":"render"}}}`}, @@ -127,6 +133,7 @@ func TestSearchInvalidInput(t *testing.T) { for _, args := range [][]string{ {}, {" "}, {strings.Repeat("x", 2049)}, {"a", "b"}, {"test", "--provider", ""}, {"test", "--max-results", "0"}, {"test", "--max-results", "101"}, + {"test", "-o", "yaml"}, {"get", "srch_test", "-o", "yaml"}, {"contents", "srch_test", "--limit", "1", "-o", "yaml"}, {"--request", "null"}, {"--request", "[]"}, {"--request", "{"}, {"--request", `{}`}, {"--request", `{"query":1}`}, {"test", "--request", `{"query":"test"}`}, {"--request", `{}`, "--provider", "exa"}, {"--request", `{}`, "--max-results", "5"}, {"--request-file", "request.json"}, @@ -139,6 +146,9 @@ func TestSearchInvalidInput(t *testing.T) { {"contents", "srch_test", "--limit", "1", "--content-format", "html"}, {"contents", "srch_test", "--limit", "1", "--content-browser-mode", "headless"}, {"contents", "srch_test", "--limit", "1", "--content-max-chars", "-1"}, + {"contents", "srch_test", "--request", "[]"}, + {"contents", "srch_test", "--request", `{"limit":1}`, "--limit", "1"}, + {"contents", "srch_test", "--request", `{"limit":1}`, "--content-source", "browser"}, {"contents", "srch_test", "--limit", "1", "--content-browser-id", " "}, {"contents", "srch_test", "--result-ids", "res_1,"}, } {