diff --git a/CHANGELOG.md b/CHANGELOG.md index dcd892e..f40a190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Optional Test `description` in specification 0.3.0 for human-readable case context, separate from its name, suite and attempt diagnostics. + ### Changed - Clarified output-line arrays and valid omission or empty arrays when no stdout or stderr was captured, on final results and previous attempts. diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index fc0e392..06aa9ae 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -163,6 +163,10 @@ "type": "string", "minLength": 1 }, + "description": { + "description": "Human-readable description of the test case", + "type": "string" + }, "status": { "description": "Final outcome of the test case", "enum": [ "passed", "failed", "skipped", "pending", "other" ] diff --git a/spec/ctrf.md b/spec/ctrf.md index 7ed4c74..f037f3b 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -965,6 +965,17 @@ The name or title of the test case. `name` MUST be present. It MUST be a non-empty string. +### 9.4.1. `description` + +**Description:** +A human-readable description of the test case, such as its purpose or the behavior it checks. + +`name` remains the case's primary display name, and `suite` describes its grouping hierarchy. `description` provides additional context; attempt-specific failure diagnostics belong in `message` and `trace`. + +**Requirements:** +`description` is OPTIONAL. +If present, it MUST be a string. + ### 9.5. `status` **Description:** @@ -2283,6 +2294,10 @@ to this specification. "type": "string", "minLength": 1 }, + "description": { + "description": "Human-readable description of the test case", + "type": "string" + }, "status": { "description": "Final outcome of the test case", "enum": [ "passed", "failed", "skipped", "pending", "other" ] diff --git a/tests/normative/additional-properties.test.json b/tests/normative/additional-properties.test.json index a483850..c7b92e8 100644 --- a/tests/normative/additional-properties.test.json +++ b/tests/normative/additional-properties.test.json @@ -211,6 +211,50 @@ }] } } + }, + { + "description": "Description is a Test field, not an attempt history field", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py", + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "description": "Not an attempt field" + } + ] + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 0419fad..6f37ec2 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1234,6 +1234,191 @@ ] } } + }, + { + "description": "Test description rejects null", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": null, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects number", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": 42, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects boolean", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": false, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects array", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": [], + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects object", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": {}, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 01f770b..b3cfd44 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -95,6 +95,80 @@ ] } } + }, + { + "description": "Test description accepts empty string", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description accepts multiline string", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "Checks reachability.\nUses ICMP echo.", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } } ] }