fix(api): filter raw responses with --jq and end output with a newline

tea api /version --jq .version failed with "--jq requires --json", and
--json is rejected for raw responses. api is annotated tea.rawjson now:
--jq filters each successful JSON response, page by page with
--paginate, and prints strings without quotes. A non-JSON response is
an error with --jq; an error response prints its body unfiltered.

JSON responses end with a newline, so an error no longer starts on the
line of the body; on a terminal JSON is indented and colored in the key
order of the response, and other text is sanitized as before. Piped
non-JSON bodies stay byte-exact.

api gains -i for --include and --silent, a Long help modeled on gh api
(placeholders, -f/-F typing, pagination, --jq) and examples, and its
help lists --jq.

Signed-off-by: bircni <bircni@icloud.com>

internal/app/api.go

@@ -5,14 +5,17 @@ package app
 
 import (
 	"bytes"
+	"cmp"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"maps"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
-	"sort"
+	"slices"
 	"strings"
 
 	"gitea.dev/tea/internal/clierr"
@@ -108,13 +111,51 @@ func apiHeaders(values []string) (http.Header, error) {
 	return headers, nil
 }
 
+// apiLong documents tea api like gh api: endpoints, placeholders, fields, pagination and output.
+const apiLong = `Make an authenticated request to the Gitea API of the selected instance and print
+the response.
+
+The endpoint is a path under /api/v1, such as repos/{owner}/{repo}/issues, or a URL of
+the instance's API. The placeholders {owner} and {repo} take the repository of the
+current directory or --repo, and {branch} the checked-out Git branch.
+
+The method is GET, or POST when fields or --input supply a request body; -X selects
+another. Fields become query parameters of GET and HEAD requests and a JSON body
+otherwise:
+
+  -f, --raw-field name=value   a string value
+  -F, --field name=value       a typed value: true, false, null, numbers, arrays and
+                               objects are JSON, @FILE reads a file and @- stdin
+
+--paginate follows the next-page links of a GET request and prints every page on its
+own line. --jq filters each JSON response with a jq expression and prints strings
+without quotes; jq does not need to be installed.
+
+On a terminal, JSON responses are indented and colored. A response outside 2xx prints
+its body, then fails with the HTTP status.`
+
+// apiExamples are the examples of tea api.
+const apiExamples = `# The login of the authenticated account
+$ tea api user --jq .login
+# Open issues of the current repository, as number and title
+$ tea api 'repos/{owner}/{repo}/issues?state=open' --jq '.[] | "\(.number) \(.title)"'
+# Create an issue: -f sends strings, -F typed values
+$ tea api repos/{owner}/{repo}/issues -f title="Crash on start" -F labels='[3]'
+# Every page of the account's repositories
+$ tea api user/repos --paginate --jq '.[].full_name'
+# Send a JSON body from a file
+$ tea api -X PATCH repos/alice/project --input settings.json`
+
 func (a *application) apiCommand() *cobra.Command {
 	var method, input, output string
 	var raw, typed, headerValues []string
-	var include, paginate bool
+	var include, paginate, silent bool
 	var maxPages int
-	cmd := &cobra.Command{Use: "api <endpoint>", Short: "Make an authenticated Gitea API request", Annotations: textOutput("api returns raw responses"), Args: argsExact(1), RunE: func(cmd *cobra.Command, args []string) error {
+	cmd := &cobra.Command{Use: "api <endpoint>", Short: "Make an authenticated Gitea API request", Long: apiLong, Example: apiExamples, Annotations: map[string]string{"tea.text": "api returns raw responses", "tea.rawjson": "true"}, Args: argsExact(1), RunE: func(cmd *cobra.Command, args []string) error {
 		usage := func(err error) error { return clierr.Wrap(clierr.Usage, err) }
+		if silent && a.options.jq != "" {
+			return usage(errors.New("--silent and --jq are mutually exclusive"))
+		}
 		headers, err := apiHeaders(headerValues)
 		if err != nil {
 			return usage(err)
@@ -210,49 +251,102 @@ func (a *application) apiCommand() *cobra.Command {
 			}
 		}
 		return writeOutput(cmd, output, func(destination io.Writer) error {
+			response := apiResponse{paginate: paginate, query: a.options.jq}
 			if output == "" || output == "-" {
-				destination = terminal.SafeWriter(destination, a.deps.Streams.OutTTY, false)
+				response.tty = a.deps.Streams.OutTTY
+				response.color = response.tty && a.colorEnabled()
 			}
-			return (services.API{Client: target.client}).Visit(cmd.Context(), services.APIOptions{Endpoint: endpoint, Method: method, Body: body, Headers: headers, Paginate: paginate, MaxPages: maxPages}, func(response *http.Response) error {
+			return (services.API{Client: target.client}).Visit(cmd.Context(), services.APIOptions{Endpoint: endpoint, Method: method, Body: body, Headers: headers, Paginate: paginate, MaxPages: maxPages}, func(r *http.Response) error {
 				if include {
-					if _, err := fmt.Fprintf(cmd.ErrOrStderr(), "%s %s\n", response.Proto, response.Status); err != nil {
-						return err
-					}
-					keys := make([]string, 0, len(response.Header))
-					for key := range response.Header {
-						keys = append(keys, key)
-					}
-					sort.Strings(keys)
-					for _, key := range keys {
-						for _, value := range response.Header.Values(key) {
-							if _, err := fmt.Fprintf(cmd.ErrOrStderr(), "%s: %s\n", key, value); err != nil {
-								return err
-							}
-						}
-					}
-					if _, err := fmt.Fprintln(cmd.ErrOrStderr()); err != nil {
+					if err := writeResponseHead(cmd.ErrOrStderr(), r); err != nil {
 						return err
 					}
 				}
-				if _, err := io.Copy(destination, response.Body); err != nil {
+				if silent {
+					_, err := io.Copy(io.Discard, r.Body)
 					return err
 				}
-				if paginate {
-					_, err := fmt.Fprintln(destination)
-					return err
-				}
-				return nil
+				return response.write(destination, r)
 			})
 		})
 	}}
-	cmd.Flags().StringVarP(&method, "method", "X", "GET", "HTTP method")
-	cmd.Flags().StringVar(&input, "input", "", "Read request body from a file or stdin (-)")
-	cmd.Flags().StringArrayVarP(&headerValues, "header", "H", nil, "Request header: Name: value")
-	cmd.Flags().StringArrayVarP(&raw, "raw-field", "f", nil, "String request field: name=value")
-	cmd.Flags().StringArrayVarP(&typed, "field", "F", nil, "JSON request field: name=value; @PATH or @- reads file text")
-	cmd.Flags().StringVar(&output, "output-file", "", "Write the complete response to a new file")
-	cmd.Flags().BoolVar(&include, "include", false, "Print response status and headers to stderr")
-	cmd.Flags().BoolVar(&paginate, "paginate", false, "Follow scoped next-page links; separate raw pages with newlines")
-	cmd.Flags().IntVar(&maxPages, "max-pages", 1000, "Maximum pages with --paginate")
+	cmd.Flags().StringVarP(&method, "method", "X", "GET", "HTTP `method`")
+	cmd.Flags().StringVar(&input, "input", "", "Read the request body from a `file`, or stdin with -")
+	cmd.Flags().StringArrayVarP(&headerValues, "header", "H", nil, "Add a request header: `\"Name: value\"`")
+	cmd.Flags().StringArrayVarP(&raw, "raw-field", "f", nil, "Add a string field: `name=value`")
+	cmd.Flags().StringArrayVarP(&typed, "field", "F", nil, "Add a typed field: `name=value`; @FILE or @- reads a file or stdin")
+	cmd.Flags().StringVar(&output, "output-file", "", "Write the complete response to a new `file`")
+	cmd.Flags().BoolVarP(&include, "include", "i", false, "Print the response status and headers to stderr")
+	cmd.Flags().BoolVar(&silent, "silent", false, "Do not print the response body")
+	cmd.Flags().BoolVar(&paginate, "paginate", false, "Follow next-page links and print every page")
+	cmd.Flags().IntVar(&maxPages, "max-pages", 1000, "Stop --paginate after this many pages")
 	return cmd
 }
+
+// apiResponse writes response bodies for tea api: filtered by the jq query, as indented and colored
+// JSON on a terminal, otherwise as received, without terminal controls on a terminal. Output to a
+// terminal, JSON and pages end with a newline.
+type apiResponse struct {
+	tty, color, paginate bool
+	query                string
+}
+
+func (o apiResponse) write(w io.Writer, r *http.Response) error {
+	mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	isJSON := mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")
+	succeeded := r.StatusCode >= 200 && r.StatusCode < 300
+	if (o.query != "" && succeeded) || (o.tty && isJSON) {
+		data, err := io.ReadAll(r.Body)
+		if err != nil {
+			return err
+		}
+		if o.query != "" && succeeded {
+			if !json.Valid(data) {
+				return fmt.Errorf("--jq needs a JSON response, but the response is %s", cmp.Or(mediaType, "not JSON"))
+			}
+			return terminal.JSONOutput{Query: o.query, Pretty: o.tty, Color: o.color}.Write(w, json.RawMessage(data))
+		}
+		if pretty, err := terminal.IndentJSON(data, o.color); err == nil {
+			_, err = io.WriteString(w, pretty)
+			return err
+		}
+		r.Body = io.NopCloser(bytes.NewReader(data))
+	}
+	last := &lastByte{w: terminal.SafeWriter(w, o.tty, false)}
+	if _, err := io.Copy(last, r.Body); err != nil {
+		return err
+	}
+	if last.written && last.b != '\n' && (o.tty || isJSON || o.paginate) {
+		_, err := io.WriteString(w, "\n")
+		return err
+	}
+	return nil
+}
+
+// lastByte remembers the last byte written through it.
+type lastByte struct {
+	w       io.Writer
+	b       byte
+	written bool
+}
+
+func (l *lastByte) Write(p []byte) (int, error) {
+	if len(p) > 0 {
+		l.b, l.written = p[len(p)-1], true
+	}
+	return l.w.Write(p)
+}
+
+// writeResponseHead prints the status line and the sorted headers of r, then a blank line.
+func writeResponseHead(w io.Writer, r *http.Response) error {
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s %s\n", r.Proto, r.Status)
+	for _, key := range slices.Sorted(maps.Keys(r.Header)) {
+		for _, value := range r.Header.Values(key) {
+			fmt.Fprintf(&b, "%s: %s\n", key, value)
+		}
+	}
+	b.WriteString("\n")
+	_, err := io.WriteString(w, b.String())
+	return err
+}

internal/app/api_test.go

@@ -209,3 +209,65 @@ func TestAPIFieldFilesPreserveTextAndValidateStdinConsumers(t *testing.T) {
 		require.Error(t, Execute(context.Background(), deps, args))
 	}
 }
+
+func TestAPIFiltersAndFormatsJSONResponses(t *testing.T) {
+	server := newAuthenticatedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json;charset=utf-8")
+		switch r.URL.Path {
+		case "/api/v1/version":
+			_, _ = w.Write([]byte(`{"version":"28.1.0","b":1}`))
+		case "/api/v1/repos/alice/nope":
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = w.Write([]byte(`{"message":"not found"}`))
+		case "/api/v1/repos":
+			if r.URL.Query().Get("page") != "2" {
+				w.Header().Set("Link", `<?page=2>; rel="next"`)
+				_, _ = w.Write([]byte(`[{"name":"one"}]`))
+				return
+			}
+			_, _ = w.Write([]byte(`[{"name":"two"}]`))
+		default:
+			w.Header().Set("Content-Type", "text/plain")
+			_, _ = w.Write([]byte("plain"))
+		}
+	}))
+	defer server.Close()
+	for _, tc := range []struct {
+		name string
+		tty  bool
+		args []string
+		want string
+		err  string
+	}{
+		{name: "jq without --json", args: []string{"version", "--jq", ".version"}, want: "28.1.0\n"},
+		{name: "jq shorthand on a terminal", tty: true, args: []string{"version", "-q", "."}, want: "{\n  \"b\": 1,\n  \"version\": \"28.1.0\"\n}\n"},
+		{name: "jq on every page", args: []string{"repos", "--paginate", "--jq", ".[].name"}, want: "one\ntwo\n"},
+		{name: "terminal keeps key order", tty: true, args: []string{"version"}, want: "{\n  \"version\": \"28.1.0\",\n  \"b\": 1\n}\n"},
+		{name: "piped JSON ends with a newline", args: []string{"version"}, want: "{\"version\":\"28.1.0\",\"b\":1}\n"},
+		{name: "error body ends with a newline", args: []string{"repos/alice/nope"}, want: "{\"message\":\"not found\"}\n", err: "HTTP 404"},
+		{name: "error body is not filtered", args: []string{"repos/alice/nope", "--jq", ".message"}, want: "{\"message\":\"not found\"}\n", err: "HTTP 404"},
+		{name: "jq needs JSON", args: []string{"text", "--jq", "."}, err: "--jq needs a JSON response, but the response is text/plain"},
+		{name: "silent", args: []string{"version", "--silent"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			deps, out, _ := serverDeps(t, server.URL)
+			deps.Streams.OutTTY = tc.tty
+			err := Execute(context.Background(), deps, append([]string{"api"}, tc.args...))
+			if tc.err != "" {
+				require.ErrorContains(t, err, tc.err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, tc.want, withoutANSI(out.String()))
+		})
+	}
+	requireUsageBeforeRequests(t, "--silent and --jq are mutually exclusive", "api", "version", "--silent", "--jq", ".")
+	requireUsageBeforeRequests(t, "api returns raw responses; --json is not supported", "api", "version", "--json")
+}
+
+func TestAPIHelpShowsJQ(t *testing.T) {
+	deps, out, _ := testDeps(t)
+	require.NoError(t, Execute(context.Background(), deps, []string{"api", "--help"}))
+	require.Contains(t, out.String(), "\n  -q, --jq expression ")
+	require.NotContains(t, out.String(), "--json fields")
+}

internal/app/app_test.go

@@ -420,7 +420,6 @@ const (
 	pr review cancel-request
 `
 	examplesToWrite = `
-	api
 	auth login
 	auth logout
 	auth refresh

internal/app/help_topics.go

@@ -154,8 +154,8 @@ func availableCommands(cmd *cobra.Command) []commandList {
 }
 
 // helpFlags renders the flags of cmd and the global flags that apply to it: --repo, --remote and --cwd
-// unless it never selects a repository, --json and --jq when it outputs JSON, and --yes when it asks
-// for confirmation. Persistent flags of parent commands, such as --org, count as its own.
+// unless it never selects a repository, --json and --jq when it outputs JSON (--jq alone when it
+// filters raw JSON), and --yes when it asks for confirmation. Persistent flags of parent commands, such as --org, count as its own.
 func helpFlags(cmd *cobra.Command) (local, inherited string) {
 	globals := cmd.Root().PersistentFlags()
 	var own []*pflag.Flag
@@ -169,7 +169,7 @@ func helpFlags(cmd *cobra.Command) (local, inherited string) {
 	slices.SortFunc(own, func(x, y *pflag.Flag) int { return strings.Compare(x.Name, y.Name) })
 	path := cmd.CommandPath()
 	repo := !slices.ContainsFunc(noRepoCommands, func(prefix string) bool { return path == prefix || strings.HasPrefix(path, prefix+" ") })
-	shown := map[string]bool{"repo": repo, "remote": repo, "cwd": repo, "json": outputsJSON(cmd), "jq": outputsJSON(cmd), "yes": cmd.Annotations["tea.confirm"] != ""}
+	shown := map[string]bool{"repo": repo, "remote": repo, "cwd": repo, "json": outputsJSON(cmd), "jq": outputsJSON(cmd) || cmd.Annotations["tea.rawjson"] != "", "yes": cmd.Annotations["tea.confirm"] != ""}
 	var shared []*pflag.Flag
 	for _, name := range []string{"repo", "remote", "cwd", "json", "jq", "yes"} {
 		if shown[name] {

internal/terminal/output.go

@@ -125,6 +125,20 @@ func (o JSONOutput) encode(w io.Writer, value any) error {
 	return err
 }
 
+// IndentJSON formats JSON text for a terminal like Write with Pretty, but keeps the key order of the
+// text: indented, sanitized, ending with a newline and, with color, highlighted.
+func IndentJSON(data []byte, color bool) (string, error) {
+	var buf bytes.Buffer
+	if err := json.Indent(&buf, bytes.TrimSpace(data), "", "  "); err != nil {
+		return "", err
+	}
+	text := SafeText(buf.String()) + "\n"
+	if color {
+		text = colorJSON(text)
+	}
+	return text, nil
+}
+
 // JSON colors, following gh: bold blue keys, green strings, yellow booleans, cyan null.
 const (
 	jsonKeyColor    = "\x1b[1;34m"