kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool is for checking the security hardening options of the Linux kernel.
   5
   6 Author: Alexander Popov <alex.popov@linux.com>
   7
   8 This module performs unit-testing of the kconfig-hardened-check engine.
   9 """
  10
  11 # pylint: disable=missing-function-docstring,line-too-long
  12
  13 import unittest
  14 import io
  15 import sys
  16 from collections import OrderedDict
  17 import json
  18 from .engine import KconfigCheck, CmdlineCheck, SysctlCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  19 import re
  20
  21
  22 class TestEngine(unittest.TestCase):
  23     """
  24     Example test scenario:
  25
  26         # 1. prepare the checklist
  27         config_checklist = []
  28         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  29         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  30         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'sysctl_name', 'expected_3')]
  31
  32         # 2. prepare the parsed kconfig options
  33         parsed_kconfig_options = OrderedDict()
  34         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  35
  36         # 3. prepare the parsed cmdline options
  37         parsed_cmdline_options = OrderedDict()
  38         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  39
  40         # 4. prepare the parsed sysctl options
  41         parsed_sysctl_options = OrderedDict()
  42         parsed_sysctl_options['sysctl_name'] = 'expected_3'
  43
  44         # 5. prepare the kernel version
  45         kernel_version = (42, 43)
  46
  47         # 6. run the engine
  48         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version)
  49
  50         # 7. check that the results are correct
  51         result = []
  52         self.get_engine_result(config_checklist, result, 'json')
  53         self.assertEqual(...
  54     """
  55
  56     @staticmethod
  57     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version):
  58         # populate the checklist with data
  59         if parsed_kconfig_options:
  60             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  61         if parsed_cmdline_options:
  62             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  63         if parsed_sysctl_options:
  64             populate_with_data(checklist, parsed_sysctl_options, 'sysctl')
  65         if kernel_version:
  66             populate_with_data(checklist, kernel_version, 'version')
  67
  68         # now everything is ready, perform the checks
  69         perform_checks(checklist)
  70
  71         # print the table with the results
  72         print('TABLE:')
  73         for opt in checklist:
  74             opt.table_print('verbose', True) # verbose mode, with_results
  75             print()
  76             print('=' * 121)
  77
  78         # print the results in JSON
  79         print('JSON:')
  80         result = []
  81         for opt in checklist:
  82             result.append(opt.json_dump(True)) # with_results
  83         print(json.dumps(result))
  84         print()
  85
  86     @staticmethod
  87     def get_engine_result(checklist, result, result_type):
  88         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  89                f'invalid result type "{result_type}"'
  90
  91         if result_type == 'json':
  92             for opt in checklist:
  93                 result.append(opt.json_dump(True)) # with_results
  94             return
  95
  96         captured_output = io.StringIO()
  97         stdout_backup = sys.stdout
  98         sys.stdout = captured_output
  99         for opt in checklist:
 100             if result_type == 'stdout_verbose':
 101                 opt.table_print('verbose', True) # verbose mode, with_results
 102             else:
 103                 opt.table_print(None, True) # normal mode, with_results
 104         sys.stdout = stdout_backup
 105         result.append(captured_output.getvalue())
 106
 107     def test_simple_kconfig(self):
 108         # 1. prepare the checklist
 109         config_checklist = []
 110         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 111         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 112         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 113         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 114         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 115         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 116         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 117         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 118         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 119         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 120
 121         # 2. prepare the parsed kconfig options
 122         parsed_kconfig_options = OrderedDict()
 123         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 124         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 125         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 126         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 127         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 128         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 129
 130         # 3. run the engine
 131         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 132
 133         # 4. check that the results are correct
 134         result = []
 135         self.get_engine_result(config_checklist, result, 'json')
 136         self.assertEqual(
 137                 result,
 138                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 139                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 140                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 141                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 142                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 143                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 144                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 145                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 146                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 147                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 148         )
 149
 150     def test_simple_cmdline(self):
 151         # 1. prepare the checklist
 152         config_checklist = []
 153         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 154         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 155         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 156         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 157         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 158         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 159         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 160         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 161         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 162         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 163
 164         # 2. prepare the parsed cmdline options
 165         parsed_cmdline_options = OrderedDict()
 166         parsed_cmdline_options['name_1'] = 'expected_1'
 167         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 168         parsed_cmdline_options['name_5'] = ''
 169         parsed_cmdline_options['name_7'] = ''
 170         parsed_cmdline_options['name_8'] = 'off'
 171         parsed_cmdline_options['name_9'] = '0'
 172
 173         # 3. run the engine
 174         self.run_engine(config_checklist, None, parsed_cmdline_options, None, None)
 175
 176         # 4. check that the results are correct
 177         result = []
 178         self.get_engine_result(config_checklist, result, 'json')
 179         self.assertEqual(
 180                 result,
 181                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 182                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 183                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 184                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 185                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 186                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 187                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 188                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 189                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 190                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 191         )
 192
 193     def test_simple_sysctl(self):
 194         # 1. prepare the checklist
 195         config_checklist = []
 196         config_checklist += [SysctlCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 197         config_checklist += [SysctlCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 198         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 199         config_checklist += [SysctlCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 200         config_checklist += [SysctlCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 201         config_checklist += [SysctlCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 202         config_checklist += [SysctlCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 203         config_checklist += [SysctlCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 204         config_checklist += [SysctlCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 205         config_checklist += [SysctlCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 206
 207         # 2. prepare the parsed sysctl options
 208         parsed_sysctl_options = OrderedDict()
 209         parsed_sysctl_options['name_1'] = 'expected_1'
 210         parsed_sysctl_options['name_2'] = 'UNexpected_2'
 211         parsed_sysctl_options['name_5'] = ''
 212         parsed_sysctl_options['name_7'] = ''
 213         parsed_sysctl_options['name_8'] = 'off'
 214         parsed_sysctl_options['name_9'] = '0'
 215
 216         # 3. run the engine
 217         self.run_engine(config_checklist, None, None, parsed_sysctl_options, None)
 218
 219         # 4. check that the results are correct
 220         result = []
 221         self.get_engine_result(config_checklist, result, 'json')
 222         self.assertEqual(
 223                 result,
 224                 [["name_1", "sysctl", "expected_1", "decision_1", "reason_1", "OK"],
 225                  ["name_2", "sysctl", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 226                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 227                  ["name_4", "sysctl", "is not set", "decision_4", "reason_4", "OK: is not found"],
 228                  ["name_5", "sysctl", "is present", "decision_5", "reason_5", "OK: is present"],
 229                  ["name_6", "sysctl", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 230                  ["name_7", "sysctl", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 231                  ["name_8", "sysctl", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 232                  ["name_9", "sysctl", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 233                  ["name_10", "sysctl", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 234         )
 235
 236     def test_complex_or(self):
 237         # 1. prepare the checklist
 238         config_checklist = []
 239         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 240                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 241         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 242                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 243         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 244                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 245         config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
 246                                 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
 247         config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 248                                 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 249         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 250                                 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 251
 252         # 2. prepare the parsed kconfig options
 253         parsed_kconfig_options = OrderedDict()
 254         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 255         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 256         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 257         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 258         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 259         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 260         parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
 261         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 262
 263         # 3. run the engine
 264         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 265
 266         # 4. check that the results are correct
 267         result = []
 268         self.get_engine_result(config_checklist, result, 'json')
 269         self.assertEqual(
 270                 result,
 271                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 272                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 273                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 274                  ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
 275                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
 276                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
 277         )
 278
 279     def test_complex_and(self):
 280         # 1. prepare the checklist
 281         config_checklist = []
 282         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 283                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 284         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 285                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 286         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 287                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 288         config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 289                                  KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 290         config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 291                                  KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 292         config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
 293                                  KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
 294
 295         # 2. prepare the parsed kconfig options
 296         parsed_kconfig_options = OrderedDict()
 297         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 298         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 299         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 300         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 301         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 302         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 303         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 304         parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
 305         parsed_kconfig_options['CONFIG_NAME_11'] = '0'
 306         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 307
 308         # 3. run the engine
 309         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 310
 311         # 4. check that the results are correct
 312         result = []
 313         self.get_engine_result(config_checklist, result, 'json')
 314         self.assertEqual(
 315                 result,
 316                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 317                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 318                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 319                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
 320                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
 321                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
 322         )
 323
 324     def test_version(self):
 325         # 1. prepare the checklist
 326         config_checklist = []
 327         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 328                                 VersionCheck((41, 101)))]
 329         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 330                                 VersionCheck((44, 1)))]
 331         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 332                                 VersionCheck((42, 44)))]
 333         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 334                                 VersionCheck((42, 43)))]
 335
 336         # 2. prepare the parsed kconfig options
 337         parsed_kconfig_options = OrderedDict()
 338         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 339         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 340
 341         # 3. prepare the kernel version
 342         kernel_version = (42, 43)
 343
 344         # 4. run the engine
 345         self.run_engine(config_checklist, parsed_kconfig_options, None, None, kernel_version)
 346
 347         # 5. check that the results are correct
 348         result = []
 349         self.get_engine_result(config_checklist, result, 'json')
 350         self.assertEqual(
 351                 result,
 352                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 353                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 354                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 355                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 356         )
 357
 358     def test_stdout(self):
 359         # 1. prepare the checklist
 360         config_checklist = []
 361         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 362                                 AND(CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 363                                     SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')))]
 364         config_checklist += [AND(CmdlineCheck('reason_4', 'decision_4', 'name_4', 'expected_4'),
 365                                  OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 366                                     SysctlCheck('reason_6', 'decision_6', 'name_6', 'expected_6')))]
 367
 368         # 2. prepare the parsed cmdline options
 369         parsed_cmdline_options = OrderedDict()
 370         parsed_cmdline_options['name_4'] = 'expected_4'
 371
 372         # 3. prepare the parsed sysctl options
 373         parsed_sysctl_options = OrderedDict()
 374         parsed_sysctl_options['name_3'] = 'UNexpected_3'
 375         parsed_sysctl_options['name_6'] = 'UNexpected_6'
 376
 377         # 4. run the engine
 378         self.run_engine(config_checklist, None, parsed_cmdline_options, parsed_sysctl_options, None)
 379
 380         # 5. check that the results are correct
 381         json_result = []
 382         self.get_engine_result(config_checklist, json_result, 'json')
 383         self.assertEqual(
 384                 json_result,
 385                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: is not found"],
 386                  ["name_4", "cmdline", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""]]
 387         )
 388
 389         stdout_result = []
 390         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 391         stdout_result_clean = [re.sub(r'\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))', '', s) for s in stdout_result]
 392         self.assertEqual(
 393                 stdout_result_clean,
 394                 [
 395 "\
 396 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\
 397 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | FAIL: CONFIG_NAME_5 is not \"expected_5\"\
 398 "               ]
 399         )
 400
 401         stdout_result = []
 402         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 403         stdout_result_clean = [re.sub(r'\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))', '', s) for s in stdout_result]
 404         self.assertEqual(
 405                 stdout_result_clean,
 406                 [
 407 "\
 408     <<< OR >>>                                                                             | FAIL: is not found\n\
 409 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\n\
 410     <<< AND >>>                                                                            | FAIL: name_3 is not \"expected_3\"\n\
 411 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | None\n\
 412 name_3                                  |sysctl | expected_3 |decision_3|     reason_3     | FAIL: \"UNexpected_3\"\
 413 "\
 414 "\
 415     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_5 is not \"expected_5\"\n\
 416 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | None\n\
 417     <<< OR >>>                                                                             | FAIL: is not found\n\
 418 CONFIG_NAME_5                           |kconfig| expected_5 |decision_5|     reason_5     | FAIL: is not found\n\
 419 name_6                                  |sysctl | expected_6 |decision_6|     reason_6     | FAIL: \"UNexpected_6\"\
 420 "               ]
 421         )
 422
 423     def test_value_overriding(self):
 424         # 1. prepare the checklist
 425         config_checklist = []
 426         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 427         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 428         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 429
 430         # 2. prepare the parsed kconfig options
 431         parsed_kconfig_options = OrderedDict()
 432         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 433
 434         # 3. prepare the parsed cmdline options
 435         parsed_cmdline_options = OrderedDict()
 436         parsed_cmdline_options['name_2'] = 'expected_2_new'
 437
 438         # 4. prepare the parsed sysctl options
 439         parsed_sysctl_options = OrderedDict()
 440         parsed_sysctl_options['name_3'] = 'expected_3_new'
 441
 442         # 5. run the engine
 443         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 444
 445         # 6. check that the results are correct
 446         result = []
 447         self.get_engine_result(config_checklist, result, 'json')
 448         self.assertEqual(
 449                 result,
 450                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 451                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 452                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 453         )
 454
 455         # 7. override expected value and perform the checks again
 456         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 457         perform_checks(config_checklist)
 458
 459         # 8. check that the results are correct
 460         result = []
 461         self.get_engine_result(config_checklist, result, 'json')
 462         self.assertEqual(
 463                 result,
 464                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 465                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 466                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 467         )
 468
 469         # 9. override expected value and perform the checks again
 470         override_expected_value(config_checklist, "name_2", "expected_2_new")
 471         perform_checks(config_checklist)
 472
 473         # 10. check that the results are correct
 474         result = []
 475         self.get_engine_result(config_checklist, result, 'json')
 476         self.assertEqual(
 477                 result,
 478                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 479                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 480                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 481         )
 482
 483         # 11. override expected value and perform the checks again
 484         override_expected_value(config_checklist, "name_3", "expected_3_new")
 485         perform_checks(config_checklist)
 486
 487         # 12. check that the results are correct
 488         result = []
 489         self.get_engine_result(config_checklist, result, 'json')
 490         self.assertEqual(
 491                 result,
 492                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 493                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 494                  ["name_3", "sysctl", "expected_3_new", "decision_3", "reason_3", "OK"]]
 495         )