[Fuego] JSON schema and examples for Fuego's run.json format
Bird, Timothy
Tim.Bird at sony.com
Thu Jun 29 22:16:59 UTC 2017
> -----Original Message-----
> From: Daniel Sangorrin on Thursday, June 29, 2017 1:12 AM
> I have a preliminar schema for Fuego's run.json format and a couple of
> examples.
> # run.json represents the result of running a test and the associated
> metadata
>
> The schema is heavily based on Kernel CI's schemas. The main differences
> are:
> - I have removed items that we don't need at the moment (kvm_guest...)
> - I have defined several metadata items specific for fuego
> - I have renamed some keys (e.g.: duration_ms instead of time)
> - I have produced a single embedded schema instead of using references to
> other files
> - I allow nesting test sets inside test sets
> - I have added the concept of reference values
>
> You can validate the provided examples with the schema at:
> http://www.jsonschemavalidator.net/
>
> This is still a draft so any feedback or suggestion is very welcomed.
>
> Thanks,
> Daniel
>
> P.S.: I'm also working on the "unified output format" proposal but this will
> probably take me
> some time. The unified output format will be just a specification to make test
> logs
> easy to parse, and it's something different from run.json files which are a
> method to
> store results and metadata before submitting it to a server or converting it to
> a different format.
OK - I have been referring to the consistent formatting of results from Fuego
as the "unified output format". That is, what you're working on here. However,
I can see how the name could be applied to the output from tests. Maybe
we should choose different names to distinguish the 2 different items:
- fuego canonical results format (what I was calling "unified output format")
- test recommended output format
Many tests will be not under our control, so we won't be able to dictate this.
But, as with kselftest output for the kernel, we can make recommendations
and work towards consistent test log output over time, to make it is easier for Fuego (and
other test systems) to parse.
>
> P.S: I send the files attached, but for the record and for inline comments I
> paste the file contents inline.
>
> *** fuego-schema.json ***
> {
> "$schema":"http://json-schema.org/schema#",
>
> "id":"http://http://www.fuegotest.org/download/fuego_schema_v1.0.json
> ",
This seems to have an extra 'http://'
> "title":"test_suite",
> "description":"A test suite JSON object corresponding to a single Fuego job
> run or execution",
> "definitions":{
> "status":{
> "type":"string",
> "description":"The status of the execution of this test case",
> "enum":[
> "PASS",
> "FAIL",
> "SKIP",
> "ERROR"
> ],
> "default":"PASS"
> },
> "measurement":{
> "title":"measurement",
> "description":"A measurement registered by a test case",
> "type":"object",
> "properties":{
> "name":{
> "type":"string",
> "description":"The name given to this measurement"
> },
> "unit":{
> "type":"string",
> "description":"The unit of this measurement"
> },
> "measure":{
> "type":[
> "string",
> "number",
> "integer"
> ],
> "description":"The data measured during the test case execution;
> the value will be interpreted based on the $unit field"
> },
> "status":{
> "$ref":"#/definitions/status"
> },
> "reference":{
> "type":"object",
> "description":"Reference measure that is compared to a measure
> to decide the status",
> "properties":{
> "threshold":{
> "type":[
> "string",
> "number",
> "integer"
> ],
> "description":"A threshold value",
> "default":"integer"
> },
> "comparison":{
> "type":"string",
> "description":"Type of comparison",
> "enum":[
> "eq",
> "ne",
> "gt",
> "ge",
> "lt",
> "le"
> ],
> "default":"gt"
> }
> }
> }
> },
> "required":[
> "measure"
> ]
> },
> "attachment":{
> "title":"attachment",
> "description":"the location of a file",
> "type":"object",
> "properties":{
> "name":{
> "type":"string",
> "description":"The name given to the attachment"
> },
> "server_uri":{
> "type":"string",
> "description":"A URI that points to this attachment"
> },
> "path":{
> "type":"string",
> "description":"A path in the test results package that identifies this
> attachment"
> }
> },
> "required":[
> "path"
> ]
> },
> "test_case":{
> "type":"object",
> "description":"A test case is the smallest test program",
> "properties":{
> "tguid":{
> "type":"string",
> "description":"globally unique identifier for the test case"
> },
> "name":{
> "type":"string",
> "description":"The name given to this test case"
> },
> "status":{
> "$ref":"#/definitions/status"
> },
> "duration_ms":{
> "type":"number",
> "description":"The number of ms it took to execute this test case",
> "default":-1
> },
> "measurements":{
> "type":"array",
> "description":"Array of measurement objects registered by this
> test case",
> "items":{
> "$ref":"#/definitions/measurement"
> }
> },
> "attachments":{
> "type":"array",
> "description":"List of attachment objects produced by this test
> case",
> "items":{
> "$ref":"#/definitions/attachment"
> },
> "additionalItems":true
> }
> },
> "required":[
> "tguid",
> "status"
> ]
> },
> "test_set":{
> "title":"test_set",
> "description":"A test set is a group of test cases",
> "type":"object",
> "properties":{
> "tguid":{
> "type":"string",
> "description":"globally unique identifier for the test case"
> },
> "name":{
> "type":"string",
> "description":"The name given to this test set"
> },
> "duration_ms":{
> "type":"number",
> "description":"The number of ms it took to execute the test set"
> },
> "test_case":{
> "type":"array",
> "description":"The list of test case objects executed by this test
> set",
> "items":{
> "$ref":"#/definitions/test_case"
> }
> },
> "test_set":{
> "type":"array",
> "description":"Nested test set",
> "items":{
> "$ref":"#/definitions/test_set"
> }
> }
> },
> "required":[
> "tguid"
> ]
> }
> },
> "type":"object",
> "properties":{
> "version":{
> "type":"string",
> "description":"The version number of this JSON schema",
> "enum":[
> "1.0"
> ]
> },
> "name":{
> "type":"string",
> "description":"The name given to this test suite"
> },
> "tguid":{
> "type":"string",
> "description":"Fuego's test globally unique identifier"
> },
Hey - thanks! But this would only be a partial tguid. Each individual measurement
should be able to be identified by a fully-qualified tguid (which maybe consists
of <this-tguid>.<test_set>.<test_case>.<measurement>
> "status":{
> "$ref":"#/definitions/status"
> },
> "duration_ms":{
> "type":"number",
> "description":"The number of ms it took to execute the entire test
> suite",
> "default":-1
> },
> "test_set":{
> "type":"array",
> "description":"The list of test set objects, executed by this test suite",
> "items":{
> "$ref":"#/definitions/test_set"
> }
> },
> "test_case":{
> "type":"array",
> "description":"List of test case objects when they do not belong to a
> test set",
> "items":{
> "$ref":"#/definitions/test_case"
> }
> },
> "metadata":{
> "type":"object",
> "description":"Object to store Fuego parameters and other accessory
> data",
> "properties":{
> "fuego_version":{
> "type":"string",
> "description":"The version of fuego (commit id, tag, ..)"
> },
> "fuego_core_version":{
> "type":"string",
> "description":"The version of fuego core (commit id, tag, ..)"
> },
> "testsuite_version":{
> "type":"string",
> "description":"string that identifies the testsuite source code
> (commit id, tarball version..)"
> },
> "host_name":{
> "type":"string",
> "description":"The name of the host executing this test suite"
> },
> "board_type":{
> "type":"string",
> "description":"The name of the board"
> },
> "board_instance":{
> "type":"string",
> "description":"The instance identifier of the board (for LAVA)"
> },
> "compiled_on":{
> "type":"string",
> "description":"The machine where the test was compiled"
> },
> "job_name":{
> "type":"string",
> "description":"The name of the job"
> },
> "kernel_version":{
> "type":"string",
> "description":"The version of the kernel running on the board"
> },
> "start_time":{
> "type":"string",
> "description":"timestamp (YYYY-MM-ddThh:mm:SSZ) when the
> test suite execution was started",
This is a change, but it appears to be iso 8601-compliant.
I'd rather not be required to express the start time in UTC. I think
we should allow TimeZone offsets here (maybe you do). I like to
see times in a locally-relevant time format. This is one of my chief
gripes about Jenkins.
> "format":"date-time"
> },
> "test_plan":{
> "type":"string",
> "description":"The test plan used"
> },
> "test_spec":{
> "type":"string",
> "description":"The test spec used"
> },
> "build_number":{
> "type":"number",
> "description":"The build number"
> },
> "keep_log":{
> "type":"boolean",
> "description":"The keep_log flag"
> },
> "reboot":{
> "type":"boolean",
> "description":"The reboot flag"
> },
> "rebuild":{
> "type":"boolean",
> "description":"The rebuild flag"
> },
> "target_precleanup":{
> "type":"boolean",
> "description":"The target_precleanup flag"
> },
> "target_postcleanup":{
> "type":"boolean",
> "description":"The target_postcleanup flag"
> },
> "workspace":{
> "type":"string",
> "description":"The workspace path"
> },
> "attachments":{
> "type":"array",
> "description":"List of attachment objects produced by this test
> suite",
> "items":{
> "$ref":"#/definitions/attachment"
> },
> "additionalItems":true
> }
> },
> "required":[
> "fuego_version",
> "host_name",
> "board_type",
> "job_name",
> "test_spec"
> ]
> }
> },
> "required":[
> "version",
> "tguid",
> "status",
> "metadata"
> ]
> }
>
> *** fuego-run-example-dhrystone.json ***
> {
> "version":"1.0",
> "tguid":"34234acdddac9234110000accvbb",
> "name":"Benchmark.Dhrystone",
> "status":"PASS",
> "duration_ms":13425,
> "test_case":[
> {
> "tguid":"ab2344cfa34546564s234acfddddaaa2342",
I want the tguid to be human readable, and invariant over runs.
In this case, I think the tguid for the single-measurement item
should be "Dhrystone.Dhrystone"
> "name":"Dhrystone",
> "status":"PASS",
> "duration_ms":13400,
> "measurements":[
> {
> "measure":1111111.1,
> "unit":"Dhrystones per Second",
> "status":"PASS",
> "reference":{
> "threshold":"1",
> "comparison":"gt"
> }
> }
> ]
> }
> ],
> "metadata":{
> "fuego_version":"0803ee7a2c5a59af159c01f561fc8d30d3d40e75",
Can we do these with 'git describe' strings instead?
ex: "fuego_version":"v1.1-24-g5ad677b"
> "fuego_core_version":"b98ae34f3c2199c46cc229ca209fc39faded938b",
> "fuego_core_version":"b98ae34f3c2199c46cc229ca209fc39faded938b",
I assume this duplication is a typo.
> "host_name":"daniel_machine_3",
> "board_type":"bbb",
> "compiled_on":"daniel_machine_3.docker",
> "job_name":"docker.default.Benchmark.Dhrystone",
> "kernel_version":"3.8.13-bone79",
> "test_plan":"none",
> "test_spec":"default",
> "start_time":"2017-06-26T09:03:04Z",
Is this actually in UTC, rather than localtime?
Where are you getting this from? date? python time.time?
> "toolchain":"qemu-armv7hf",
> "build_number":4,
> "keep_log":true,
> "reboot":false,
> "rebuild":true,
> "target_precleanup":true,
> "target_postcleanup":true,
> "workspace":"/fuego-rw/buildzone",
> "attachments":[
> {
> "name":"devlog",
> "path":"devlog.txt"
> },
> {
> "name":"devlog",
> "path":"devlog.txt"
> },
> {
> "name":"syslog.before",
> "path":"syslog.before.txt"
> },
> {
> "name":"syslog.after",
> "path":"syslog.after.txt"
> },
> {
> "name":"testlog",
> "path":"testlog.txt"
> },
> {
> "name":"consolelog",
> "path":"consolelog.txt"
> },
> {
> "name":"test_spec",
> "path":"spec.json"
> }
> ]
> }
> }
>
> *** fuego-run-example-bonnie.json ***
> {
> "version":"1.0",
> "tguid":"334453ascdddac92341100ssd00acc3vbb",
> "name":"Benchmark.bonnie",
> "status":"PASS",
> "duration_ms":118352,
> "test_set":[
> {
> "tguid":"234534546564s234acfddddaaa2342",
> "name":"Random_Create",
> "status":"PASS",
> "test_case":[
> {
> "tguid":"123a34546564s234acfddddaaa2342",
> "name":"Create",
tguid would be: "bonnie.Random_Create.Create"
I think we can synthesize the tguid from the name fields. If it's unambiguous
how to generate an invariant name for a test case/measure, then there's
no need to have a separate field for the tguid.
The idea with TGUIDs is that we can compare them between test systems.
As an example, I'd like to be able to compare a 0-day Dhrystone.Dhrystone value
for the latest kernel with a Fuego Dhrystone.Dhrystone value for the LTSI
kernel.
> "status":"PASS",
> "measurements":[
> {
> "measure":6050
> }
> ]
> },
> {
> "tguid":"234567234cfa34546564s234acfddddaaa2342",
> "name":"Delete",
> "status":"PASS",
> "measurements":[
> {
> "measure":12695
> }
> ]
> }
> ]
> },
> {
> "tguid":"234534546564s234acfddddaaa2342",
> "name":"Sequential_Create",
> "status":"PASS",
> "test_case":[
> {
> "tguid":"123423a34546564s234acfddddaaa2342",
> "name":"Create",
> "status":"PASS",
> "measurements":[
> {
> "measure":9563
> }
> ]
> },
> {
> "tguid":"5612342357234cfa34546564s234acfddddaaa2342",
> "name":"Delete",
> "status":"PASS",
> "measurements":[
> {
> "measure":13379
> }
> ]
> }
> ]
> },
> {
> "tguid":"235434546564s234acfddddaaa2342",
> "name":"Sequential_Input",
> "status":"PASS",
> "test_case":[
> {
> "tguid":"123213454a34546564s234acfddddaaa2342",
> "name":"PerChr",
> "status":"PASS",
> "measurements":[
> {
> "measure":6581
> }
> ]
> }
> ]
> },
> {
> "tguid":"234534546564s234acfddddaaa2342",
> "name":"Sequential_Output",
> "status":"PASS",
> "test_case":[
> {
> "tguid":"123a23453434546564s234acfddddaaa2342",
> "name":"Block",
> "status":"PASS",
> "measurements":[
> {
> "measure":0
> }
> ]
> },
> {
> "tguid":"245345567234cfa34546564s234acfddddaaa2342",
> "name":"PerChr",
> "status":"PASS",
> "measurements":[
> {
> "measure":6538
> }
> ]
> },
> {
> "tguid":"656345567234cfa34546564s234acfddddaaa2342",
> "name":"Rewrite",
> "status":"PASS",
> "measurements":[
> {
> "measure":0
> }
> ]
> }
> ]
> }
> ],
> "metadata":{
> "fuego_version":"0803ee7a2c5a59af159c01f561fc8d30d3d40e75",
> "fuego_core_version":"b98ae34f3c2199c46cc229ca209fc39faded938b",
> "testsuite_version":"b98ae34f3c2199c46cc229ca209fc39faded938b",
> "host_name":"daniel_machine_3",
> "board_type":"bbb",
> "compiled_on":"daniel_machine_3.docker",
> "job_name":"docker.default.Benchmark.bonnie",
> "kernel_version":"3.8.13-bone79",
> "test_plan":"none",
> "test_spec":"default",
> "start_time":"2017-06-29T06:43:43Z",
> "toolchain":"qemu-armv7hf",
> "build_number":1,
> "keep_log":true,
> "reboot":false,
> "rebuild":true,
> "target_precleanup":true,
> "target_postcleanup":true,
> "workspace":"/fuego-rw/buildzone",
> "attachments":[
> {
> "name":"devlog",
> "path":"devlog.txt"
> },
> {
> "name":"devlog",
> "path":"devlog.txt"
> },
> {
> "name":"syslog.before",
> "path":"syslog.before.txt"
> },
> {
> "name":"syslog.after",
> "path":"syslog.after.txt"
> },
> {
> "name":"testlog",
> "path":"testlog.txt"
> },
> {
> "name":"consolelog",
> "path":"consolelog.txt"
> },
> {
> "name":"test_spec",
> "path":"spec.json"
> }
> ]
> }
> }
>
>
>
>
>
More information about the Fuego
mailing list