9d756e8011
By running validations with arguments intentionally chosen to make the validation requirements impossible to fulfill, we can check how the framework handles failures, and if the tested validations do not produce false positives. The functional test now, in addition to the existing set of checks, allow for optional 'negative_results' testing. Variables entries for all tested validations are checked for the presence of 'negative_results' dictionary key. If it is found, the 'extra_args' key contents within are used to run the validation in question, after it has passed the test for a successful validation run. The 'negative results' run is expected to fail, failure is supressed, and result recorded. Passing result, conversely, is interpreted as an error, because validation passing shouldn't be possible with the arguments supplied. Show validations test was patched to proper form of the command. Signed-off-by: Jiri Podivin <jpodivin@redhat.com> Change-Id: If7e7b25b9c876184bd04f3f0ddefdee20172a5b4
65 lines
1.8 KiB
YAML
65 lines
1.8 KiB
YAML
---
|
||
- name: Execute validation commands
|
||
shell:
|
||
cmd: "{{ validation_command }}"
|
||
executable: /bin/bash
|
||
|
||
- name: set fact for Validation action
|
||
set_fact: v_action="{{ action }}"
|
||
|
||
- name: Get Run results - Positive - these are supposed to pass
|
||
block:
|
||
- name: Get run results
|
||
register: result
|
||
shell:
|
||
cmd: "cat {{ val_output }}"
|
||
executable: /bin/bash
|
||
|
||
- name: Get json data
|
||
set_fact:
|
||
jsondata: "{{ result.stdout | from_json }}"
|
||
|
||
- name: Get Validations Status
|
||
set_fact:
|
||
status: "{{ jsondata | json_query(jsonres) }}"
|
||
vars:
|
||
jsonres: 'results[*].Status'
|
||
|
||
- fail:
|
||
msg: "Validation failed with {{ validation_status }}: some of the validations have failed. {{ status }}"
|
||
when: validation_status != "PASSED"
|
||
loop: "{{ status }}"
|
||
loop_control:
|
||
loop_var: validation_status
|
||
when: v_action == 'run'
|
||
|
||
- name: Get Run results - Negative - these are supposed to fail
|
||
# This task should fail with return code != 0
|
||
# The validation is supplied with parameters that make it impossible to pass.
|
||
block:
|
||
- name: Get run results
|
||
register: result
|
||
shell:
|
||
cmd: "cat {{ val_output }}"
|
||
executable: /bin/bash
|
||
|
||
- name: Get json data
|
||
set_fact:
|
||
jsondata: "{{ result.stdout | from_json }}"
|
||
|
||
- name: Get Validations Status
|
||
set_fact:
|
||
status: "{{ jsondata | json_query(jsonres) }}"
|
||
vars:
|
||
jsonres: 'results[*].Status'
|
||
|
||
- fail:
|
||
msg: "Validation passed with {{ validation_status }} when it shouldn't have: some of the validations have passed. {{ status }}"
|
||
when: validation_status == "PASSED"
|
||
loop: "{{ status }}"
|
||
loop_control:
|
||
loop_var: validation_status
|
||
when:
|
||
- v_action == 'run'
|
||
- "'negative_results' in name.value"
|