diff --git a/cmd/kosli/get.go b/cmd/kosli/get.go index bc3cdd8a0..3dfcbbbfd 100644 --- a/cmd/kosli/get.go +++ b/cmd/kosli/get.go @@ -32,6 +32,7 @@ func newGetCmd(out io.Writer) *cobra.Command { newGetServiceAccountCmd(out), newGetDefaultOrgCmd(out), newGetRegoPolicyCmd(out), + newGetEvaluationCmd(out), ) return cmd } diff --git a/cmd/kosli/getEvaluation.go b/cmd/kosli/getEvaluation.go new file mode 100644 index 000000000..444948bb9 --- /dev/null +++ b/cmd/kosli/getEvaluation.go @@ -0,0 +1,381 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/kosli-dev/cli/internal/evaluations" + "github.com/kosli-dev/cli/internal/output" + "github.com/kosli-dev/cli/internal/regopolicies" + "github.com/kosli-dev/cli/internal/requests" + "github.com/spf13/cobra" +) + +const getEvaluationShortDesc = `Get a server-side evaluation, or what it was given.` + +const getEvaluationLongDesc = getEvaluationShortDesc + ` +The evaluation is shown with its status, the policy it ran, the trails it was +asked about, its outcome when it completed, the error when it failed, and the +decision it recorded, if any. The command exits 0 whatever the outcome. + +With ^--show-input^ and ^--output json^, the document the policy read as ^input^ is +added under ^.input^: each trail as it stood at the evaluation's request time. + +With ^--fixture^ and ^--dir^, what the evaluation was given is written to the +directory, so it can be replayed with stock OPA: + - ^input.json^: the document the policy read as ^input^. + - ^data.json^: the params, which the policy reads as ^data.params^. + - ^policy/^: the files of the policy that ran, at their paths in the bundle. +The directory must be empty or not exist yet. An evaluation that is still +pending can be written too, as its input does not depend on its outcome.` + +const getEvaluationExample = ` +# get an evaluation: +kosli get evaluation yourEvaluationID \ + --api-token yourAPIToken \ + --org yourOrgName + +# get an evaluation with the input its policy read: +kosli get evaluation yourEvaluationID \ + --show-input \ + --output json \ + --api-token yourAPIToken \ + --org yourOrgName + +# write what an evaluation was given to ./replay, and replay it with OPA: +kosli get evaluation yourEvaluationID \ + --fixture \ + --dir ./replay \ + --api-token yourAPIToken \ + --org yourOrgName +opa eval --input replay/input.json --data replay/data.json \ + --bundle replay/policy/ 'data.policy.allow' +` + +type getEvaluationOptions struct { + output string + showInput bool + fixture bool + dir string +} + +func newGetEvaluationCmd(out io.Writer) *cobra.Command { + o := new(getEvaluationOptions) + cmd := &cobra.Command{ + Use: "evaluation EVALUATION-ID", + Short: getEvaluationShortDesc, + Long: getEvaluationLongDesc, + Example: getEvaluationExample, + Args: cobra.ExactArgs(1), + Annotations: map[string]string{betaCLIAnnotation: ""}, + PreRunE: func(cmd *cobra.Command, args []string) error { + err := RequireGlobalFlags(global, []string{"Org", "ApiToken"}) + if err != nil { + return ErrorBeforePrintingUsage(cmd, err.Error()) + } + if err := o.validate(); err != nil { + return ErrorBeforePrintingUsage(cmd, err.Error()) + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if o.fixture { + return o.runFixture(out, args[0]) + } + return o.run(out, args[0]) + }, + } + + cmd.Flags().StringVarP(&o.output, "output", "o", "table", outputFlag) + cmd.Flags().BoolVar(&o.showInput, "show-input", false, evaluationShowInputFlag) + cmd.Flags().BoolVar(&o.fixture, "fixture", false, evaluationFixtureFlag) + cmd.Flags().StringVar(&o.dir, "dir", "", evaluationFixtureDirFlag) + + return cmd +} + +// validate refuses a combination before anything is read, so a refusal never +// leaves a half-written fixture behind. +func (o *getEvaluationOptions) validate() error { + if o.fixture && o.showInput { + return fmt.Errorf("only one of --fixture or --show-input can be given") + } + if o.showInput && o.output != "json" { + return fmt.Errorf("--show-input needs --output json") + } + if o.fixture && o.dir == "" { + return fmt.Errorf("--fixture needs --dir") + } + if o.dir != "" && !o.fixture { + return fmt.Errorf("--dir needs --fixture") + } + if o.fixture { + return requireEmptyDir(o.dir) + } + return nil +} + +func (o *getEvaluationOptions) run(out io.Writer, id string) error { + client := evaluations.NewClient(kosliClient, global.Host, global.ApiToken, global.DryRun) + evaluation, err := client.Get(global.Org, id) + if err != nil { + return err + } + raw := evaluation.Raw + if o.showInput { + input, err := client.Input(global.Org, id) + if err != nil { + return err + } + raw, err = withInput(raw, input) + if err != nil { + return err + } + } + + return output.FormattedPrint(raw, o.output, out, 0, + map[string]output.FormatOutputFunc{ + "table": printEvaluationAsTable, + "json": output.PrintJson, + }) +} + +// withInput adds the input as the evaluation's last key, leaving the server's +// body as it was sent, keys in their order, rather than re-encoding it. +func withInput(evaluation, input string) (string, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(evaluation), &fields); err != nil { + return "", fmt.Errorf("failed to parse the evaluation: %w", err) + } + if !json.Valid([]byte(input)) { + return "", fmt.Errorf("failed to parse the evaluation's input") + } + body := strings.TrimSuffix(strings.TrimSpace(evaluation), "}") + return body + `,"input":` + input + "}", nil +} + +// evaluationRead is what the table needs from an evaluation; -o json prints +// the server's body instead. +type evaluationRead struct { + ID string `json:"id"` + Status string `json:"status"` + RequestedAt float64 `json:"requested_at"` + RecordedAt float64 `json:"recorded_at"` + Context struct { + Trails []evaluations.TrailRef `json:"trails"` + } `json:"context"` + Policy struct { + Identifier string `json:"identifier"` + Version int `json:"version"` + Digest string `json:"digest"` + Files map[string]string `json:"files"` + } `json:"policy"` + Params map[string]any `json:"params"` + Result *evaluations.Result `json:"result"` + Failure *evaluations.Failure `json:"error"` + OPAVersion string `json:"opa_version"` + Decision *evaluations.Decision `json:"decision"` + DecisionAttestationID string `json:"decision_attestation_id"` +} + +func printEvaluationAsTable(raw string, out io.Writer, page int) error { + var evaluation evaluationRead + if err := json.Unmarshal([]byte(raw), &evaluation); err != nil { + return err + } + + policy := map[string]any{"identifier": evaluation.Policy.Identifier, "digest": evaluation.Policy.Digest} + if evaluation.Policy.Identifier != "" { + policy["version"] = float64(evaluation.Policy.Version) + } + trails := make([]string, len(evaluation.Context.Trails)) + for i, trail := range evaluation.Context.Trails { + trails[i] = trail.Flow + "/" + trail.Trail + } + rows := []string{ + fmt.Sprintf("ID:\t%s", evaluation.ID), + fmt.Sprintf("Status:\t%s", evaluation.Status), + fmt.Sprintf("Policy:\t%s", citedPolicyText(policy)), + fmt.Sprintf("Trails:\t%s", strings.Join(trails, ", ")), + } + for _, field := range []struct { + label string + at float64 + }{ + {"Requested at", evaluation.RequestedAt}, + {"Recorded at", evaluation.RecordedAt}, + } { + at, err := formattedTimestamp(field.at, true) + if err != nil { + return err + } + rows = append(rows, fmt.Sprintf("%s:\t%s", field.label, at)) + } + + if evaluation.Status == evaluations.StatusCompleted && evaluation.Result != nil { + verdict := serverVerdict(evaluation.Result) + outcome := "DENIED" + if verdict.Allow { + outcome = "ALLOWED" + } + rows = append(rows, fmt.Sprintf("Outcome:\t%s", outcome)) + violations := verdict.Violations + if listed, ok := verdict.Outputs["violations"]; ok { + violations = nil + for _, violation := range violationList(listed) { + violations = append(violations, violationText(violation)) + } + } + for i, violation := range violations { + label := "" + if i == 0 { + label = "Violations:" + } + rows = append(rows, fmt.Sprintf("%s\t%s", label, violation)) + } + } + if evaluation.Failure != nil { + rows = append(rows, fmt.Sprintf("Error:\t%s: %s", evaluation.Failure.Kind, evaluation.Failure.Message)) + } + if evaluation.OPAVersion != "" { + rows = append(rows, fmt.Sprintf("OPA version:\t%s", evaluation.OPAVersion)) + } + if evaluation.Decision != nil { + rows = append(rows, fmt.Sprintf("Decision:\t%s for control %s in %s/%s", + evaluation.Decision.Name, evaluation.Decision.Control, + evaluation.Decision.Flow, evaluation.Decision.Trail)) + } + if evaluation.DecisionAttestationID != "" { + rows = append(rows, fmt.Sprintf("Decision attestation:\t%s", evaluation.DecisionAttestationID)) + } + + tabFormattedPrint(out, []string{}, rows) + return nil +} + +func (o *getEvaluationOptions) runFixture(out io.Writer, id string) error { + client := evaluations.NewClient(kosliClient, global.Host, global.ApiToken, global.DryRun) + read, err := client.Get(global.Org, id) + if err != nil { + return err + } + var evaluation evaluationRead + if err := json.Unmarshal([]byte(read.Raw), &evaluation); err != nil { + return fmt.Errorf("failed to parse the evaluation: %w", err) + } + input, err := client.Input(global.Org, id) + if err != nil { + return err + } + files, err := policyFiles(evaluation) + if err != nil { + return err + } + + params := evaluation.Params + if params == nil { + params = map[string]any{} + } + data, err := json.MarshalIndent(map[string]any{"params": params}, "", " ") + if err != nil { + return err + } + var indented bytes.Buffer + if err := json.Indent(&indented, []byte(input), "", " "); err != nil { + return fmt.Errorf("failed to parse the evaluation's input: %w", err) + } + + written := map[string][]byte{ + "input.json": indented.Bytes(), + "data.json": data, + } + for name, content := range files { + if !filepath.IsLocal(name) { + return fmt.Errorf("%q is not a path inside the policy directory", name) + } + written[filepath.Join("policy", name)] = []byte(content) + } + for name, content := range written { + path := filepath.Join(o.dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return err + } + } + + _, err = fmt.Fprintf(out, "Wrote the input, params and policy of evaluation %s to %s\n", id, o.dir) + return err +} + +// policyFiles is the bundle that ran: an inline policy's files come with the +// evaluation, and a published version's are read from the rego policy. +func policyFiles(evaluation evaluationRead) (map[string]string, error) { + if evaluation.Policy.Identifier == "" { + return evaluation.Policy.Files, nil + } + endpoint, err := regopolicies.BundleURL(global.Host, global.Org, + evaluation.Policy.Identifier, evaluation.Policy.Version) + if err != nil { + return nil, err + } + response, err := kosliClient.Do(&requests.RequestParams{ + Method: http.MethodGet, + URL: endpoint, + Token: global.ApiToken, + }) + if err != nil { + return nil, err + } + return untarBundle([]byte(response.Body)) +} + +func untarBundle(archive []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + return nil, fmt.Errorf("failed to read the policy bundle: %w", err) + } + files := map[string]string{} + reader := tar.NewReader(zipped) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return files, nil + } + if err != nil { + return nil, fmt.Errorf("failed to read the policy bundle: %w", err) + } + if header.Typeflag != tar.TypeReg { + continue + } + content, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read the policy bundle: %w", err) + } + files[header.Name] = string(content) + } +} + +func requireEmptyDir(dir string) error { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if len(entries) > 0 { + return fmt.Errorf("%s is not empty; choose an empty or new directory", dir) + } + return nil +} diff --git a/cmd/kosli/getEvaluation_test.go b/cmd/kosli/getEvaluation_test.go new file mode 100644 index 000000000..81d2c76cb --- /dev/null +++ b/cmd/kosli/getEvaluation_test.go @@ -0,0 +1,427 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + evaluationPath = "/api/v2/evaluations/test-org/01EVAL" + evaluationInputPath = evaluationPath + "/input" + publishedBundlePath = "/api/v2/rego-policies/test-org/pr-approved/versions/3/bundle" + + readCompletedPublished = `{"id":"01EVAL","status":"completed","requested_at":1757000000.5,` + + `"recorded_at":1757000000.5,"context":{"trails":[{"flow":"my-flow","trail":"my-trail"}]},` + + `"policy":{"identifier":"pr-approved","version":3,"digest":"sha256:abc123"},` + + `"params":{"min_approvals":2},"result":{"allow":true,"outputs":{}},"opa_version":"1.4.0",` + + `"decision":{"control":"ctrl-1","name":"ctrl-1-decision","flow":"my-flow","trail":"my-trail"},` + + `"decision_attestation_id":"01DECISION"}` + readDeniedInline = `{"id":"01EVAL","status":"completed","requested_at":1757000000.5,` + + `"recorded_at":1757000000.5,"context":{"trails":[{"flow":"my-flow","trail":"my-trail"}]},` + + `"policy":{"digest":"sha256:def456","files":{"policy.rego":"package policy\n",` + + `"lib/util.rego":"package util\n"}},"params":{},` + + `"result":{"allow":false,"outputs":{"violations":["change is not approved"]}}}` + readFailed = `{"id":"01EVAL","status":"failed","requested_at":1757000000.5,` + + `"recorded_at":1757000000.5,"context":{"trails":[{"flow":"my-flow","trail":"my-trail"}]},` + + `"policy":{"digest":"sha256:def456","files":{"policy.rego":"package policy\n"}},"params":{},` + + `"error":{"kind":"policy_error","message":"allow is undefined"}}` + readPending = `{"id":"01EVAL","status":"pending","requested_at":1757000000.5,` + + `"recorded_at":1757000000.5,"context":{"trails":[{"flow":"my-flow","trail":"my-trail"}]},` + + `"policy":{"digest":"sha256:def456","files":{"policy.rego":"package policy\n"}},"params":{}}` + + evaluationInput = `{"evaluation":{"time":"2025-09-04T15:33:20.500000Z"},` + + `"trails":[{"flow_name":"my-flow","trail_name":"my-trail"}],` + + `"trail":{"flow_name":"my-flow","trail_name":"my-trail"}}` +) + +type GetEvaluationCommandTestSuite struct { + suite.Suite +} + +func (suite *GetEvaluationCommandTestSuite) SetupTest() { + // Makes formattedTimestamp print a fixed UTC time. + suite.T().Setenv("KOSLI_TESTS", "true") +} + +func (suite *GetEvaluationCommandTestSuite) getCmd(host, args string) string { + return fmt.Sprintf("get evaluation %s --host %s --org test-org --api-token test-token --max-api-retries 0", + args, host) +} + +func (suite *GetEvaluationCommandTestSuite) TestItIsBeta() { + root, err := newRootCmd(io.Discard, io.Discard, nil) + require.NoError(suite.T(), err) + cmd, _, err := root.Find([]string{"get", "evaluation"}) + require.NoError(suite.T(), err) + + require.Equal(suite.T(), "evaluation", cmd.Name()) + require.Contains(suite.T(), cmd.Annotations, betaCLIAnnotation) +} + +func (suite *GetEvaluationCommandTestSuite) TestItNeedsTheOrgAndApiToken() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + noConfig := withoutOrgAndApiToken(suite.T()) + + _, combined, _, _, err := executeCommandC(fmt.Sprintf( + "get evaluation 01EVAL --api-token test-token --host %s %s", server.URL, noConfig)) + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "Error: --org is not set\n") + require.Empty(suite.T(), fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestItTakesExactlyOneId() { + for _, args := range []string{"", "01EVAL 01OTHER"} { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, args)) + + require.Error(suite.T(), err, args) + require.Contains(suite.T(), combined, "accepts 1 arg(s)", args) + require.Empty(suite.T(), fake.reads, args) + } +} + +func (suite *GetEvaluationCommandTestSuite) TestItPrintsACompletedEvaluationOfAPublishedPolicy() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL")) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), []string{evaluationPath}, fake.reads) + for _, row := range [][2]string{ + {"ID", "01EVAL"}, + {"Status", "completed"}, + {"Policy", "pr-approved, version 3 (sha256:abc123)"}, + {"Trails", "my-flow/my-trail"}, + {"Requested at", "Sat, 16 Jan 2016 00:00:00 UTC"}, + {"Recorded at", "Sat, 16 Jan 2016 00:00:00 UTC"}, + {"Outcome", "ALLOWED"}, + {"OPA version", "1.4.0"}, + {"Decision", "ctrl-1-decision for control ctrl-1 in my-flow/my-trail"}, + {"Decision attestation", "01DECISION"}, + } { + requireRow(suite.T(), combined, row[0], row[1]) + } +} + +func (suite *GetEvaluationCommandTestSuite) TestItPrintsADenialWithItsViolationsAndAnInlinePolicyByDigest() { + server, _ := newFakeEvaluationRead(suite.T(), readDeniedInline) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL")) + + require.NoError(suite.T(), err, "reading a denial is not a denial") + requireRow(suite.T(), combined, "Policy", "sha256:def456") + requireRow(suite.T(), combined, "Outcome", "DENIED") + requireRow(suite.T(), combined, "Violations", "change is not approved") + require.NotContains(suite.T(), combined, "Decision") +} + +func (suite *GetEvaluationCommandTestSuite) TestItPrintsWhyAnEvaluationFailedAndExitsZero() { + server, _ := newFakeEvaluationRead(suite.T(), readFailed) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL")) + + require.NoError(suite.T(), err) + requireRow(suite.T(), combined, "Status", "failed") + requireRow(suite.T(), combined, "Error", "policy_error: allow is undefined") + require.NotContains(suite.T(), combined, "Outcome") +} + +func (suite *GetEvaluationCommandTestSuite) TestItPrintsAPendingEvaluationWithNoOutcome() { + server, _ := newFakeEvaluationRead(suite.T(), readPending) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL")) + + require.NoError(suite.T(), err) + requireRow(suite.T(), combined, "Status", "pending") + require.NotContains(suite.T(), combined, "Outcome") +} + +func (suite *GetEvaluationCommandTestSuite) TestItPrintsTheEvaluationAsJSONAsTheServerSentIt() { + server, _ := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, _, stdout, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --output json")) + + require.NoError(suite.T(), err) + require.JSONEq(suite.T(), readCompletedPublished, stdout) +} + +func (suite *GetEvaluationCommandTestSuite) TestItReturnsANotFoundInTheServersWords() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + fake.evaluation.status = http.StatusNotFound + fake.evaluation.body = `{"message":"Evaluation '01EVAL' does not exist in org 'test-org'"}` + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL")) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "Evaluation '01EVAL' does not exist in org 'test-org'") +} + +func (suite *GetEvaluationCommandTestSuite) TestItKeepsTheIdInOneSegment() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, _, _, _, _ = executeCommandC(suite.getCmd(server.URL, "a/b")) + + require.Equal(suite.T(), []string{"/api/v2/evaluations/test-org/a%2Fb"}, fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestItRefusesAnIdThatIsNotASegment() { + for _, id := range []string{".", ".."} { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "'"+id+"'")) + + require.Error(suite.T(), err, id) + require.Contains(suite.T(), combined, fmt.Sprintf("%q is not a valid evaluation id", id), id) + require.Empty(suite.T(), fake.reads, id) + } +} + +func (suite *GetEvaluationCommandTestSuite) TestShowInputAddsTheInputUnderItsOwnKey() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, _, stdout, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --show-input --output json")) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), []string{evaluationPath, evaluationInputPath}, fake.reads) + var printed map[string]any + require.NoError(suite.T(), json.Unmarshal([]byte(stdout), &printed)) + require.JSONEq(suite.T(), evaluationInput, mustJSON(suite.T(), printed["input"])) + delete(printed, "input") + require.JSONEq(suite.T(), readCompletedPublished, mustJSON(suite.T(), printed)) +} + +func (suite *GetEvaluationCommandTestSuite) TestShowInputNeedsJSON() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --show-input")) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "--show-input needs --output json") + require.Empty(suite.T(), fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestAnInputThatCannotBeBuiltIsReportedInTheServersWords() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + fake.input.status = http.StatusConflict + fake.input.body = `{"message":"No policy input can be built: trail has more than one attestation named 'tests'."}` + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --show-input -o json")) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "No policy input can be built") +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureOfAnInlinePolicyWritesInputParamsAndFiles() { + server, fake := newFakeEvaluationRead(suite.T(), readDeniedInline) + dir := filepath.Join(suite.T().TempDir(), "replay") + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), []string{evaluationPath, evaluationInputPath}, fake.reads) + require.JSONEq(suite.T(), evaluationInput, readFile(suite.T(), dir, "input.json")) + require.JSONEq(suite.T(), `{"params":{}}`, readFile(suite.T(), dir, "data.json")) + require.Equal(suite.T(), "package policy\n", readFile(suite.T(), dir, "policy/policy.rego")) + require.Equal(suite.T(), "package util\n", readFile(suite.T(), dir, "policy/lib/util.rego")) + require.Contains(suite.T(), combined, "Wrote the input, params and policy of evaluation 01EVAL to "+dir) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureOfAPublishedPolicyUnpacksItsBundle() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + fake.bundle = fakeRoute{body: gzippedTar(suite.T(), map[string]string{ + "policy.rego": "package policy\n", + "data.yaml": "threshold: 1\n", + "lib/util.rego": "package util\n", + })} + dir := filepath.Join(suite.T().TempDir(), "replay") + + _, _, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.NoError(suite.T(), err) + require.Equal(suite.T(), []string{evaluationPath, evaluationInputPath, publishedBundlePath}, fake.reads) + require.JSONEq(suite.T(), `{"params":{"min_approvals":2}}`, readFile(suite.T(), dir, "data.json")) + require.Equal(suite.T(), "package policy\n", readFile(suite.T(), dir, "policy/policy.rego")) + require.Equal(suite.T(), "threshold: 1\n", readFile(suite.T(), dir, "policy/data.yaml")) + require.Equal(suite.T(), "package util\n", readFile(suite.T(), dir, "policy/lib/util.rego")) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureOfAPendingEvaluationIsWritten() { + server, _ := newFakeEvaluationRead(suite.T(), readPending) + dir := filepath.Join(suite.T().TempDir(), "replay") + + _, _, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.NoError(suite.T(), err) + require.JSONEq(suite.T(), evaluationInput, readFile(suite.T(), dir, "input.json")) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureNeedsADirectory() { + server, fake := newFakeEvaluationRead(suite.T(), readDeniedInline) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture")) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "--fixture needs --dir") + require.Empty(suite.T(), fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestADirectoryNeedsAFixture() { + server, fake := newFakeEvaluationRead(suite.T(), readDeniedInline) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --dir "+suite.T().TempDir())) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "--dir needs --fixture") + require.Empty(suite.T(), fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureAndShowInputAreRefusedTogether() { + server, fake := newFakeEvaluationRead(suite.T(), readDeniedInline) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, + "01EVAL --fixture --dir "+suite.T().TempDir()+" --show-input -o json")) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, "only one of --fixture or --show-input can be given") + require.Empty(suite.T(), fake.reads) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureIsNotWrittenIntoADirectoryThatHasFiles() { + server, fake := newFakeEvaluationRead(suite.T(), readDeniedInline) + dir := suite.T().TempDir() + require.NoError(suite.T(), os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("mine"), 0o644)) + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, dir+" is not empty") + require.Empty(suite.T(), fake.reads) + require.Equal(suite.T(), "mine", readFile(suite.T(), dir, "keep.txt")) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureRefusesAPolicyFileOutsideTheDirectory() { + escaping := `{"id":"01EVAL","status":"completed","requested_at":1.0,"recorded_at":1.0,` + + `"context":{"trails":[]},"policy":{"digest":"sha256:def456",` + + `"files":{"../escaped.rego":"package policy\n"}},"params":{}}` + server, _ := newFakeEvaluationRead(suite.T(), escaping) + parent := suite.T().TempDir() + dir := filepath.Join(parent, "replay") + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, `"../escaped.rego" is not a path inside the policy directory`) + require.NoFileExists(suite.T(), filepath.Join(dir, "escaped.rego")) +} + +func (suite *GetEvaluationCommandTestSuite) TestAFixtureRefusesABundleEntryOutsideTheDirectory() { + server, fake := newFakeEvaluationRead(suite.T(), readCompletedPublished) + fake.bundle = fakeRoute{body: gzippedTar(suite.T(), map[string]string{ + "../escaped.rego": "package policy\n", + })} + dir := filepath.Join(suite.T().TempDir(), "replay") + + _, combined, _, _, err := executeCommandC(suite.getCmd(server.URL, "01EVAL --fixture --dir "+dir)) + + require.Error(suite.T(), err) + require.Contains(suite.T(), combined, `"../escaped.rego" is not a path inside the policy directory`) + require.NoFileExists(suite.T(), filepath.Join(dir, "escaped.rego")) +} + +// fakeEvaluationRead answers the evaluation, input and bundle routes with +// their body and status, 200 unless set, and records the escaped path of every +// request. Anything else is a 404. +type fakeEvaluationRead struct { + evaluation fakeRoute + input fakeRoute + bundle fakeRoute + reads []string +} + +type fakeRoute struct { + body string + status int +} + +func newFakeEvaluationRead(t *testing.T, evaluation string) (*httptest.Server, *fakeEvaluationRead) { + t.Helper() + fake := &fakeEvaluationRead{ + evaluation: fakeRoute{body: evaluation}, + input: fakeRoute{body: evaluationInput}, + bundle: fakeRoute{status: http.StatusNotFound}, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fake.reads = append(fake.reads, r.URL.EscapedPath()) + route := fakeRoute{status: http.StatusNotFound} + switch r.URL.EscapedPath() { + case evaluationPath: + route = fake.evaluation + case evaluationInputPath: + route = fake.input + case publishedBundlePath: + route = fake.bundle + } + if route.status == 0 { + route.status = http.StatusOK + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(route.status) + _, _ = io.WriteString(w, route.body) + })) + t.Cleanup(server.Close) + return server, fake +} + +func gzippedTar(t *testing.T, files map[string]string) string { + t.Helper() + var archive bytes.Buffer + zipped := gzip.NewWriter(&archive) + tw := tar.NewWriter(zipped) + for name, body := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, zipped.Close()) + return archive.String() +} + +// requireRow asserts that a table row holds the label and the value, however +// wide the label column is. +func requireRow(t *testing.T, table, label, value string) { + t.Helper() + require.Regexp(t, "(?m)^"+regexp.QuoteMeta(label)+`:\s+`+regexp.QuoteMeta(value)+"$", table) +} + +func readFile(t *testing.T, dir, name string) string { + t.Helper() + content, err := os.ReadFile(filepath.Join(dir, name)) + require.NoError(t, err) + return string(content) +} + +func mustJSON(t *testing.T, value any) string { + t.Helper() + encoded, err := json.Marshal(value) + require.NoError(t, err) + return string(encoded) +} + +func TestGetEvaluationCommandTestSuite(t *testing.T) { + suite.Run(t, new(GetEvaluationCommandTestSuite)) +} diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index d8218cf06..27bf13b2a 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -165,6 +165,9 @@ Paths the list already matches stay excluded whatever is later added there, so k regoPolicyCommentFlag = "[optional] Why this version is published. It is kept with the version." regoPolicyVersionFlag = "[optional] The version of the rego policy to get. Without it, the policy and its current version are shown." regoPolicyDownloadFlag = "[optional] The file to write the bundle of the version given by --policy-version to, as a gzipped tar. An existing file is not overwritten." + evaluationShowInputFlag = "[optional] Add the document the policy read as input under .input. Needs --output json." + evaluationFixtureFlag = "[optional] Write the evaluation's input, params and policy files to --dir, to replay it with OPA." + evaluationFixtureDirFlag = "[conditional] The directory --fixture writes to. It must be empty or not exist yet. Only needed with --fixture." regoPolicySearchFlag = "[optional] Only list rego policies whose name or identifier contains this substring (case-insensitive)." regoPolicySortFlag = "[optional] The field to sort rego policies by. Valid values are: [updated_at, identifier]. (defaults to updated_at)" regoPolicySortDirectionFlag = "[optional] The direction to sort rego policies in. Valid values are: [asc, desc]. (defaults to desc)" diff --git a/cmd/kosli/testdata/empty-flag-audit-coverage.json b/cmd/kosli/testdata/empty-flag-audit-coverage.json index 51bd5e8e6..add13efd9 100644 --- a/cmd/kosli/testdata/empty-flag-audit-coverage.json +++ b/cmd/kosli/testdata/empty-flag-audit-coverage.json @@ -675,6 +675,12 @@ "get environment": { "output": "string" }, + "get evaluation": { + "dir": "string", + "fixture": "bool", + "output": "string", + "show-input": "bool" + }, "get flow": { "output": "string" }, diff --git a/hack/empty-flag-audit/spec.json b/hack/empty-flag-audit/spec.json index 18bdc5fc2..ead54a938 100644 --- a/hack/empty-flag-audit/spec.json +++ b/hack/empty-flag-audit/spec.json @@ -1581,6 +1581,29 @@ "setup": [], "verify": [] }, + "get evaluation": { + "args": [ + "01EVALUATIONPROBE" + ], + "flags": {}, + "baseline_ok": false, + "error": "the server the audit runs against does not run server-side evaluations", + "needs": "server-side evaluation", + "flags_to_test": [ + "dir", + "fixture", + "output", + "show-input" + ], + "flag_values": { + "dir": "probe-replay", + "fixture": "true", + "output": "json", + "show-input": "true" + }, + "setup": [], + "verify": [] + }, "list rego-policies": { "args": [], "flags": {}, diff --git a/internal/evaluations/wait.go b/internal/evaluations/wait.go index 3ce542e84..2275cdb68 100644 --- a/internal/evaluations/wait.go +++ b/internal/evaluations/wait.go @@ -31,13 +31,12 @@ type WaitOptions = poll.Options // caller can point at the evaluation that is still running. type StillPendingError struct { ID string - URL string Waited time.Duration } func (e *StillPendingError) Error() string { - return fmt.Sprintf("evaluation %s is still pending after %s; read it later with GET %s", - e.ID, e.Waited, e.URL) + return fmt.Sprintf("evaluation %s is still pending after %s; read it later with 'kosli get evaluation %s'", + e.ID, e.Waited, e.ID) } // Get reads one evaluation. A pending one is a normal answer, not an error. @@ -70,10 +69,6 @@ func (c *Client) Get(org, id string) (*Evaluation, error) { // the budget expires. It returns an evaluation only when that evaluation is // terminal, so a caller never has to ask whether the answer it holds is final. func (c *Client) WaitForTerminal(ctx context.Context, org, id string, options WaitOptions) (*Evaluation, error) { - endpoint, err := evaluationURL(c.host, org, id) - if err != nil { - return nil, err - } evaluation, err := poll.Until(ctx, options.Or(DefaultWaitOptions), func() (*Evaluation, bool, error) { evaluation, err := c.Get(org, id) @@ -85,11 +80,38 @@ func (c *Client) WaitForTerminal(ctx context.Context, org, id string, options Wa var expired *poll.ExpiredError if errors.As(err, &expired) { - return nil, &StillPendingError{ID: id, URL: endpoint, Waited: expired.Waited} + return nil, &StillPendingError{ID: id, Waited: expired.Waited} } return evaluation, err } +// evaluationURL escapes the id into one path segment, so an id holding a slash +// cannot name another route. It refuses "." and "..", which a URL path +// resolves however they are escaped. func evaluationURL(host, org, id string) (string, error) { - return url.JoinPath(host, "api/v2/evaluations", org, id) + switch id { + case "", ".", "..": + return "", fmt.Errorf("%q is not a valid evaluation id", id) + } + return url.JoinPath(host, "api/v2/evaluations/"+url.PathEscape(org)+"/"+url.PathEscape(id)) +} + +// Input reads the document the evaluation's policy reads as input, as the +// server sent it. Its 409 says the input cannot be built, which no retry +// changes. +func (c *Client) Input(org, id string) (string, error) { + endpoint, err := evaluationURL(c.host, org, id) + if err != nil { + return "", err + } + response, err := c.http.Do(&requests.RequestParams{ + Method: http.MethodGet, + URL: endpoint + "/input", + Token: c.token, + DisableConflictRetry: true, + }) + if err != nil { + return "", err + } + return response.Body, nil } diff --git a/internal/evaluations/wait_test.go b/internal/evaluations/wait_test.go index 1b4c5fe9b..b5de1d5d5 100644 --- a/internal/evaluations/wait_test.go +++ b/internal/evaluations/wait_test.go @@ -189,7 +189,6 @@ func TestWaitGivesUpWithTheEvaluationIdAndNoVerdict(t *testing.T) { var stillPending *StillPendingError require.True(t, errors.As(err, &stillPending), "expected a still-pending error, got %T", err) require.Equal(t, "01JABCDEF", stillPending.ID) - require.Equal(t, server.URL+"/api/v2/evaluations/my-org/01JABCDEF", stillPending.URL) // The time actually spent, not the budget: a read carries no deadline of // its own, so the two can differ and only the first answers "did the // platform meet its promise". @@ -200,7 +199,7 @@ func TestWaitGivesUpWithTheEvaluationIdAndNoVerdict(t *testing.T) { "no nanosecond noise in a sentence about a budget") require.Contains(t, err.Error(), "01JABCDEF") require.Contains(t, err.Error(), "still pending") - require.Contains(t, err.Error(), "read it later with GET "+server.URL+"/api/v2/evaluations/my-org/01JABCDEF") + require.Contains(t, err.Error(), "read it later with 'kosli get evaluation 01JABCDEF'") require.NotContains(t, err.Error(), "denied") }