kernel_hardening_checker/test_engine.py

   1 #!/usr/bin/env python3
   2
   3 """
   4 This tool is for checking the security hardening options of the Linux kernel.
   5
   6 Author: Alexander Popov <alex.popov@linux.com>
   7
   8 This module performs unit-testing of the kernel-hardening-checker engine.
   9 """
  10
  11 # pylint: disable=missing-function-docstring,line-too-long
  12
  13 import unittest
  14 import io
  15 import sys
  16 from collections import OrderedDict
  17 import json
  18 import inspect
  19 from .engine import KconfigCheck, CmdlineCheck, SysctlCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  20
  21
  22 class TestEngine(unittest.TestCase):
  23     """
  24     Example test scenario:
  25
  26         # 1. prepare the checklist
  27         config_checklist = []
  28         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  29         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  30         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'sysctl_name', 'expected_3')]
  31
  32         # 2. prepare the parsed kconfig options
  33         parsed_kconfig_options = OrderedDict()
  34         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  35
  36         # 3. prepare the parsed cmdline options
  37         parsed_cmdline_options = OrderedDict()
  38         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  39
  40         # 4. prepare the parsed sysctl options
  41         parsed_sysctl_options = OrderedDict()
  42         parsed_sysctl_options['sysctl_name'] = 'expected_3'
  43
  44         # 5. prepare the kernel version
  45         kernel_version = (42, 43)
  46
  47         # 6. run the engine
  48         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version)
  49
  50         # 7. check that the results are correct
  51         result = []
  52         self.get_engine_result(config_checklist, result, 'json')
  53         self.assertEqual(...
  54     """
  55
  56     @staticmethod
  57     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version):
  58         # populate the checklist with data
  59         if parsed_kconfig_options:
  60             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  61         if parsed_cmdline_options:
  62             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  63         if parsed_sysctl_options:
  64             populate_with_data(checklist, parsed_sysctl_options, 'sysctl')
  65         if kernel_version:
  66             populate_with_data(checklist, kernel_version, 'version')
  67
  68         # now everything is ready, perform the checks
  69         perform_checks(checklist)
  70
  71         # print the table with the results
  72         print(f'\n{inspect.stack()[1].function}():')
  73         print('=' * 121)
  74         for opt in checklist:
  75             opt.table_print('verbose', True) # verbose mode, with_results
  76             print()
  77             print('=' * 121)
  78
  79         # print the results in JSON
  80         result = []
  81         for opt in checklist:
  82             result.append(opt.json_dump(True)) # with_results
  83         print(json.dumps(result))
  84         print()
  85
  86     @staticmethod
  87     def get_engine_result(checklist, result, result_type):
  88         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  89                f'invalid result type "{result_type}"'
  90
  91         if result_type == 'json':
  92             for opt in checklist:
  93                 result.append(opt.json_dump(True)) # with_results
  94             return
  95
  96         captured_output = io.StringIO()
  97         stdout_backup = sys.stdout
  98         sys.stdout = captured_output
  99         for opt in checklist:
 100             if result_type == 'stdout_verbose':
 101                 opt.table_print('verbose', True) # verbose mode, with_results
 102             else:
 103                 opt.table_print(None, True) # normal mode, with_results
 104         sys.stdout = stdout_backup
 105         result.append(captured_output.getvalue())
 106
 107     def test_simple_kconfig(self):
 108         # 1. prepare the checklist
 109         config_checklist = []
 110         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 111         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 112         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 113         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 114         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 115         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 116         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 117         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 118         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 119         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 120
 121         # 2. prepare the parsed kconfig options
 122         parsed_kconfig_options = OrderedDict()
 123         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 124         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 125         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 126         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 127         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 128         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 129
 130         # 3. run the engine
 131         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 132
 133         # 4. check that the results are correct
 134         result = []
 135         self.get_engine_result(config_checklist, result, 'json')
 136         self.assertEqual(
 137                 result,
 138                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 139                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 140                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 141                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 142                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 143                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 144                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 145                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 146                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 147                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 148         )
 149
 150     def test_simple_cmdline(self):
 151         # 1. prepare the checklist
 152         config_checklist = []
 153         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 154         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 155         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 156         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 157         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 158         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 159         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 160         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 161         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 162         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 163
 164         # 2. prepare the parsed cmdline options
 165         parsed_cmdline_options = OrderedDict()
 166         parsed_cmdline_options['name_1'] = 'expected_1'
 167         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 168         parsed_cmdline_options['name_5'] = ''
 169         parsed_cmdline_options['name_7'] = ''
 170         parsed_cmdline_options['name_8'] = 'off'
 171         parsed_cmdline_options['name_9'] = '0'
 172
 173         # 3. run the engine
 174         self.run_engine(config_checklist, None, parsed_cmdline_options, None, None)
 175
 176         # 4. check that the results are correct
 177         result = []
 178         self.get_engine_result(config_checklist, result, 'json')
 179         self.assertEqual(
 180                 result,
 181                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 182                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 183                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 184                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 185                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 186                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 187                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 188                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 189                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 190                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 191         )
 192
 193     def test_simple_sysctl(self):
 194         # 1. prepare the checklist
 195         config_checklist = []
 196         config_checklist += [SysctlCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 197         config_checklist += [SysctlCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 198         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 199         config_checklist += [SysctlCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 200         config_checklist += [SysctlCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 201         config_checklist += [SysctlCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 202         config_checklist += [SysctlCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 203         config_checklist += [SysctlCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 204         config_checklist += [SysctlCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 205         config_checklist += [SysctlCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 206
 207         # 2. prepare the parsed sysctl options
 208         parsed_sysctl_options = OrderedDict()
 209         parsed_sysctl_options['name_1'] = 'expected_1'
 210         parsed_sysctl_options['name_2'] = 'UNexpected_2'
 211         parsed_sysctl_options['name_5'] = ''
 212         parsed_sysctl_options['name_7'] = ''
 213         parsed_sysctl_options['name_8'] = 'off'
 214         parsed_sysctl_options['name_9'] = '0'
 215
 216         # 3. run the engine
 217         self.run_engine(config_checklist, None, None, parsed_sysctl_options, None)
 218
 219         # 4. check that the results are correct
 220         result = []
 221         self.get_engine_result(config_checklist, result, 'json')
 222         self.assertEqual(
 223                 result,
 224                 [["name_1", "sysctl", "expected_1", "decision_1", "reason_1", "OK"],
 225                  ["name_2", "sysctl", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 226                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 227                  ["name_4", "sysctl", "is not set", "decision_4", "reason_4", "OK: is not found"],
 228                  ["name_5", "sysctl", "is present", "decision_5", "reason_5", "OK: is present"],
 229                  ["name_6", "sysctl", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 230                  ["name_7", "sysctl", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 231                  ["name_8", "sysctl", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 232                  ["name_9", "sysctl", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 233                  ["name_10", "sysctl", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 234         )
 235
 236     def test_complex_or(self):
 237         # 1. prepare the checklist
 238         config_checklist = []
 239         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 240                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 241         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 242                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 243         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 244                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 245         config_checklist += [OR(KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'expected_7'),
 246                                 KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not set'))]
 247         config_checklist += [OR(KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'expected_9'),
 248                                 KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is present'))]
 249         config_checklist += [OR(KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'expected_11'),
 250                                 KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'is not off'))]
 251
 252         # 2. prepare the parsed kconfig options
 253         parsed_kconfig_options = OrderedDict()
 254         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 255         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 256         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 257         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 258         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 259         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 260         parsed_kconfig_options['CONFIG_NAME_10'] = 'UNexpected_10'
 261         parsed_kconfig_options['CONFIG_NAME_12'] = 'really_not_off'
 262
 263         # 3. run the engine
 264         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 265
 266         # 4. check that the results are correct
 267         result = []
 268         self.get_engine_result(config_checklist, result, 'json')
 269         self.assertEqual(
 270                 result,
 271                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 272                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 273                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 274                  ["CONFIG_NAME_7", "kconfig", "expected_7", "decision_7", "reason_7", "OK: CONFIG_NAME_8 is not found"],
 275                  ["CONFIG_NAME_9", "kconfig", "expected_9", "decision_9", "reason_9", "OK: CONFIG_NAME_10 is present"],
 276                  ["CONFIG_NAME_11", "kconfig", "expected_11", "decision_11", "reason_11", "OK: CONFIG_NAME_12 is not off"]]
 277         )
 278
 279     def test_complex_and(self):
 280         # 1. prepare the checklist
 281         config_checklist = []
 282         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 283                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 284         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 285                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 286         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 287                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 288         config_checklist += [AND(KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'expected_7'),
 289                                  KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is present'))]
 290         config_checklist += [AND(KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'expected_9'),
 291                                  KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off'))]
 292         config_checklist += [AND(KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'expected_11'),
 293                                  KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'is not off'))]
 294
 295         # 2. prepare the parsed kconfig options
 296         parsed_kconfig_options = OrderedDict()
 297         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 298         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 299         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 300         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 301         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 302         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 303         parsed_kconfig_options['CONFIG_NAME_7'] = 'expected_7'
 304         parsed_kconfig_options['CONFIG_NAME_9'] = 'expected_9'
 305         parsed_kconfig_options['CONFIG_NAME_10'] = '0'
 306         parsed_kconfig_options['CONFIG_NAME_11'] = 'expected_11'
 307
 308         # 3. run the engine
 309         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 310
 311         # 4. check that the results are correct
 312         result = []
 313         self.get_engine_result(config_checklist, result, 'json')
 314         self.assertEqual(
 315                 result,
 316                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 317                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 318                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 319                  ["CONFIG_NAME_7", "kconfig", "expected_7", "decision_7", "reason_7", "FAIL: CONFIG_NAME_8 is not present"],
 320                  ["CONFIG_NAME_9", "kconfig", "expected_9", "decision_9", "reason_9", "FAIL: CONFIG_NAME_10 is off"],
 321                  ["CONFIG_NAME_11", "kconfig", "expected_11", "decision_11", "reason_11", "FAIL: CONFIG_NAME_12 is off, not found"]]
 322         )
 323
 324     def test_complex_nested(self):
 325         # 1. prepare the checklist
 326         config_checklist = []
 327         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 328                                  OR(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 329                                     KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')))]
 330         config_checklist += [AND(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 331                                  OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 332                                     KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6')))]
 333         config_checklist += [OR(KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'expected_7'),
 334                                  AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 335                                      KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'expected_9')))]
 336         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 337                                  AND(KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'expected_11'),
 338                                      KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12')))]
 339
 340         # 2. prepare the parsed kconfig options
 341         parsed_kconfig_options = OrderedDict()
 342         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 343         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 344         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 345         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 346         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 347         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 348         parsed_kconfig_options['CONFIG_NAME_7'] = 'UNexpected_7'
 349         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 350         parsed_kconfig_options['CONFIG_NAME_9'] = 'expected_9'
 351         parsed_kconfig_options['CONFIG_NAME_10'] = 'UNexpected_10'
 352         parsed_kconfig_options['CONFIG_NAME_11'] = 'UNexpected_11'
 353         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 354
 355         # 3. run the engine
 356         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 357
 358         # 4. check that the results are correct
 359         result = []
 360         self.get_engine_result(config_checklist, result, 'json')
 361         self.assertEqual(
 362                 result,
 363                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 364                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""],
 365                  ["CONFIG_NAME_7", "kconfig", "expected_7", "decision_7", "reason_7", "OK: CONFIG_NAME_8 is \"expected_8\""],
 366                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: \"UNexpected_10\""]]
 367         )
 368
 369     def test_version(self):
 370         # 1. prepare the checklist
 371         config_checklist = []
 372         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 373                                 VersionCheck((41, 101)))]
 374         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 375                                 VersionCheck((44, 1)))]
 376         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 377                                 VersionCheck((42, 44)))]
 378         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 379                                 VersionCheck((42, 43)))]
 380
 381         # 2. prepare the parsed kconfig options
 382         parsed_kconfig_options = OrderedDict()
 383         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 384         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 385
 386         # 3. prepare the kernel version
 387         kernel_version = (42, 43)
 388
 389         # 4. run the engine
 390         self.run_engine(config_checklist, parsed_kconfig_options, None, None, kernel_version)
 391
 392         # 5. check that the results are correct
 393         result = []
 394         self.get_engine_result(config_checklist, result, 'json')
 395         self.assertEqual(
 396                 result,
 397                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 398                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 399                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 400                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 401         )
 402
 403     def test_stdout(self):
 404         # 1. prepare the checklist
 405         config_checklist = []
 406         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 407                                 CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 408                                 SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3'))]
 409         config_checklist += [AND(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 410                                  CmdlineCheck('reason_5', 'decision_5', 'name_5', 'expected_5'),
 411                                  SysctlCheck('reason_6', 'decision_6', 'name_6', 'expected_6'))]
 412
 413         # 2. prepare the parsed kconfig options
 414         parsed_kconfig_options = OrderedDict()
 415         parsed_kconfig_options['CONFIG_NAME_1'] = 'UNexpected_1'
 416
 417         # 3. prepare the parsed cmdline options
 418         parsed_cmdline_options = OrderedDict()
 419         parsed_cmdline_options['name_2'] = 'expected_2'
 420         parsed_cmdline_options['name_5'] = 'UNexpected_5'
 421
 422         # 4. prepare the parsed sysctl options
 423         parsed_sysctl_options = OrderedDict()
 424         parsed_sysctl_options['name_6'] = 'expected_6'
 425
 426         # 5. run the engine
 427         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 428
 429         # 6. check that the results are correct
 430         json_result = []
 431         self.get_engine_result(config_checklist, json_result, 'json')
 432         self.assertEqual(
 433                 json_result,
 434                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: name_2 is \"expected_2\""],
 435                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "FAIL: name_5 is not \"expected_5\""]]
 436         )
 437
 438         stdout_result = []
 439         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 440         self.assertEqual(
 441                 stdout_result,
 442                 [
 443 "\
 444 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | \x1b[32mOK: name_2 is \"expected_2\"\x1b[0m\
 445 CONFIG_NAME_4                           |kconfig| expected_4 |decision_4|     reason_4     | \x1b[31mFAIL: name_5 is not \"expected_5\"\x1b[0m\
 446 "               ]
 447         )
 448
 449         stdout_result = []
 450         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 451         self.assertEqual(
 452                 stdout_result,
 453                 [
 454 "\
 455     <<< OR >>>                                                                             | \x1b[32mOK: name_2 is \"expected_2\"\x1b[0m\n\
 456 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | \x1b[31mFAIL: \"UNexpected_1\"\x1b[0m\n\
 457 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | \x1b[32mOK\x1b[0m\n\
 458 name_3                                  |sysctl | expected_3 |decision_3|     reason_3     | None\
 459 "\
 460 "\
 461     <<< AND >>>                                                                            | \x1b[31mFAIL: name_5 is not \"expected_5\"\x1b[0m\n\
 462 CONFIG_NAME_4                           |kconfig| expected_4 |decision_4|     reason_4     | None\n\
 463 name_5                                  |cmdline| expected_5 |decision_5|     reason_5     | \x1b[31mFAIL: \"UNexpected_5\"\x1b[0m\n\
 464 name_6                                  |sysctl | expected_6 |decision_6|     reason_6     | \x1b[32mOK\x1b[0m\
 465 "               ]
 466         )
 467
 468     def test_value_overriding(self):
 469         # 1. prepare the checklist
 470         config_checklist = []
 471         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 472         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 473         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 474
 475         # 2. prepare the parsed kconfig options
 476         parsed_kconfig_options = OrderedDict()
 477         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 478
 479         # 3. prepare the parsed cmdline options
 480         parsed_cmdline_options = OrderedDict()
 481         parsed_cmdline_options['name_2'] = 'expected_2_new'
 482
 483         # 4. prepare the parsed sysctl options
 484         parsed_sysctl_options = OrderedDict()
 485         parsed_sysctl_options['name_3'] = 'expected_3_new'
 486
 487         # 5. run the engine
 488         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 489
 490         # 6. check that the results are correct
 491         result = []
 492         self.get_engine_result(config_checklist, result, 'json')
 493         self.assertEqual(
 494                 result,
 495                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 496                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 497                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 498         )
 499
 500         # 7. override expected value and perform the checks again
 501         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 502         perform_checks(config_checklist)
 503
 504         # 8. check that the results are correct
 505         result = []
 506         self.get_engine_result(config_checklist, result, 'json')
 507         self.assertEqual(
 508                 result,
 509                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 510                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 511                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 512         )
 513
 514         # 9. override expected value and perform the checks again
 515         override_expected_value(config_checklist, "name_2", "expected_2_new")
 516         perform_checks(config_checklist)
 517
 518         # 10. check that the results are correct
 519         result = []
 520         self.get_engine_result(config_checklist, result, 'json')
 521         self.assertEqual(
 522                 result,
 523                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 524                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 525                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 526         )
 527
 528         # 11. override expected value and perform the checks again
 529         override_expected_value(config_checklist, "name_3", "expected_3_new")
 530         perform_checks(config_checklist)
 531
 532         # 12. check that the results are correct
 533         result = []
 534         self.get_engine_result(config_checklist, result, 'json')
 535         self.assertEqual(
 536                 result,
 537                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 538                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 539                  ["name_3", "sysctl", "expected_3_new", "decision_3", "reason_3", "OK"]]
 540         )