kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool is for checking the security hardening options of the Linux kernel.
   5
   6 Author: Alexander Popov <alex.popov@linux.com>
   7
   8 This module performs unit-testing of the kconfig-hardened-check engine.
   9 """
  10
  11 # pylint: disable=missing-function-docstring,line-too-long
  12
  13 import unittest
  14 import io
  15 import sys
  16 from collections import OrderedDict
  17 import json
  18 from .engine import KconfigCheck, CmdlineCheck, SysctlCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  19
  20
  21 class TestEngine(unittest.TestCase):
  22     """
  23     Example test scenario:
  24
  25         # 1. prepare the checklist
  26         config_checklist = []
  27         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  28         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  29         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'sysctl_name', 'expected_3')]
  30
  31         # 2. prepare the parsed kconfig options
  32         parsed_kconfig_options = OrderedDict()
  33         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  34
  35         # 3. prepare the parsed cmdline options
  36         parsed_cmdline_options = OrderedDict()
  37         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  38
  39         # 4. prepare the parsed sysctl options
  40         parsed_sysctl_options = OrderedDict()
  41         parsed_sysctl_options['sysctl_name'] = 'expected_3'
  42
  43         # 5. prepare the kernel version
  44         kernel_version = (42, 43)
  45
  46         # 6. run the engine
  47         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version)
  48
  49         # 7. check that the results are correct
  50         result = []
  51         self.get_engine_result(config_checklist, result, 'json')
  52         self.assertEqual(...
  53     """
  54
  55     @staticmethod
  56     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, kernel_version):
  57         # populate the checklist with data
  58         if parsed_kconfig_options:
  59             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  60         if parsed_cmdline_options:
  61             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  62         if parsed_sysctl_options:
  63             populate_with_data(checklist, parsed_sysctl_options, 'sysctl')
  64         if kernel_version:
  65             populate_with_data(checklist, kernel_version, 'version')
  66
  67         # now everything is ready, perform the checks
  68         perform_checks(checklist)
  69
  70         # print the table with the results
  71         print('TABLE:')
  72         for opt in checklist:
  73             opt.table_print('verbose', True) # verbose mode, with_results
  74             print()
  75             print('=' * 121)
  76
  77         # print the results in JSON
  78         print('JSON:')
  79         result = []
  80         for opt in checklist:
  81             result.append(opt.json_dump(True)) # with_results
  82         print(json.dumps(result))
  83         print()
  84
  85     @staticmethod
  86     def get_engine_result(checklist, result, result_type):
  87         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  88                f'invalid result type "{result_type}"'
  89
  90         if result_type == 'json':
  91             for opt in checklist:
  92                 result.append(opt.json_dump(True)) # with_results
  93             return
  94
  95         captured_output = io.StringIO()
  96         stdout_backup = sys.stdout
  97         sys.stdout = captured_output
  98         for opt in checklist:
  99             if result_type == 'stdout_verbose':
 100                 opt.table_print('verbose', True) # verbose mode, with_results
 101             else:
 102                 opt.table_print(None, True) # normal mode, with_results
 103         sys.stdout = stdout_backup
 104         result.append(captured_output.getvalue())
 105
 106     def test_simple_kconfig(self):
 107         # 1. prepare the checklist
 108         config_checklist = []
 109         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 110         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 111         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 112         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 113         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 114         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 115         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 116         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 117         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 118         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 119
 120         # 2. prepare the parsed kconfig options
 121         parsed_kconfig_options = OrderedDict()
 122         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 123         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 124         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 125         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 126         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 127         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 128
 129         # 3. run the engine
 130         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 131
 132         # 4. check that the results are correct
 133         result = []
 134         self.get_engine_result(config_checklist, result, 'json')
 135         self.assertEqual(
 136                 result,
 137                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 138                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 139                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 140                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 141                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 142                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 143                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 144                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 145                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 146                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 147         )
 148
 149     def test_simple_cmdline(self):
 150         # 1. prepare the checklist
 151         config_checklist = []
 152         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 153         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 154         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 155         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 156         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 157         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 158         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 159         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 160         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 161         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 162
 163         # 2. prepare the parsed cmdline options
 164         parsed_cmdline_options = OrderedDict()
 165         parsed_cmdline_options['name_1'] = 'expected_1'
 166         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 167         parsed_cmdline_options['name_5'] = ''
 168         parsed_cmdline_options['name_7'] = ''
 169         parsed_cmdline_options['name_8'] = 'off'
 170         parsed_cmdline_options['name_9'] = '0'
 171
 172         # 3. run the engine
 173         self.run_engine(config_checklist, None, parsed_cmdline_options, None, None)
 174
 175         # 4. check that the results are correct
 176         result = []
 177         self.get_engine_result(config_checklist, result, 'json')
 178         self.assertEqual(
 179                 result,
 180                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 181                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 182                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 183                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 184                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 185                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 186                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 187                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 188                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 189                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 190         )
 191
 192     def test_simple_sysctl(self):
 193         # 1. prepare the checklist
 194         config_checklist = []
 195         config_checklist += [SysctlCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 196         config_checklist += [SysctlCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 197         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 198         config_checklist += [SysctlCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 199         config_checklist += [SysctlCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 200         config_checklist += [SysctlCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 201         config_checklist += [SysctlCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 202         config_checklist += [SysctlCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 203         config_checklist += [SysctlCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 204         config_checklist += [SysctlCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 205
 206         # 2. prepare the parsed sysctl options
 207         parsed_sysctl_options = OrderedDict()
 208         parsed_sysctl_options['name_1'] = 'expected_1'
 209         parsed_sysctl_options['name_2'] = 'UNexpected_2'
 210         parsed_sysctl_options['name_5'] = ''
 211         parsed_sysctl_options['name_7'] = ''
 212         parsed_sysctl_options['name_8'] = 'off'
 213         parsed_sysctl_options['name_9'] = '0'
 214
 215         # 3. run the engine
 216         self.run_engine(config_checklist, None, None, parsed_sysctl_options, None)
 217
 218         # 4. check that the results are correct
 219         result = []
 220         self.get_engine_result(config_checklist, result, 'json')
 221         self.assertEqual(
 222                 result,
 223                 [["name_1", "sysctl", "expected_1", "decision_1", "reason_1", "OK"],
 224                  ["name_2", "sysctl", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 225                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 226                  ["name_4", "sysctl", "is not set", "decision_4", "reason_4", "OK: is not found"],
 227                  ["name_5", "sysctl", "is present", "decision_5", "reason_5", "OK: is present"],
 228                  ["name_6", "sysctl", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 229                  ["name_7", "sysctl", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 230                  ["name_8", "sysctl", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 231                  ["name_9", "sysctl", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 232                  ["name_10", "sysctl", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 233         )
 234
 235     def test_complex_or(self):
 236         # 1. prepare the checklist
 237         config_checklist = []
 238         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 239                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 240         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 241                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 242         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 243                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 244         config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
 245                                 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
 246         config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 247                                 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 248         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 249                                 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 250
 251         # 2. prepare the parsed kconfig options
 252         parsed_kconfig_options = OrderedDict()
 253         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 254         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 255         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 256         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 257         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 258         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 259         parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
 260         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 261
 262         # 3. run the engine
 263         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 264
 265         # 4. check that the results are correct
 266         result = []
 267         self.get_engine_result(config_checklist, result, 'json')
 268         self.assertEqual(
 269                 result,
 270                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 271                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 272                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 273                  ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
 274                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
 275                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
 276         )
 277
 278     def test_complex_and(self):
 279         # 1. prepare the checklist
 280         config_checklist = []
 281         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 282                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 283         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 284                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 285         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 286                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 287         config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 288                                  KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 289         config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 290                                  KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 291         config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
 292                                  KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
 293
 294         # 2. prepare the parsed kconfig options
 295         parsed_kconfig_options = OrderedDict()
 296         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 297         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 298         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 299         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 300         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 301         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 302         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 303         parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
 304         parsed_kconfig_options['CONFIG_NAME_11'] = '0'
 305         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 306
 307         # 3. run the engine
 308         self.run_engine(config_checklist, parsed_kconfig_options, None, None, None)
 309
 310         # 4. check that the results are correct
 311         result = []
 312         self.get_engine_result(config_checklist, result, 'json')
 313         self.assertEqual(
 314                 result,
 315                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 316                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 317                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 318                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
 319                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
 320                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
 321         )
 322
 323     def test_version(self):
 324         # 1. prepare the checklist
 325         config_checklist = []
 326         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 327                                 VersionCheck((41, 101)))]
 328         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 329                                 VersionCheck((44, 1)))]
 330         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 331                                 VersionCheck((42, 44)))]
 332         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 333                                 VersionCheck((42, 43)))]
 334
 335         # 2. prepare the parsed kconfig options
 336         parsed_kconfig_options = OrderedDict()
 337         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 338         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 339
 340         # 3. prepare the kernel version
 341         kernel_version = (42, 43)
 342
 343         # 4. run the engine
 344         self.run_engine(config_checklist, parsed_kconfig_options, None, None, kernel_version)
 345
 346         # 5. check that the results are correct
 347         result = []
 348         self.get_engine_result(config_checklist, result, 'json')
 349         self.assertEqual(
 350                 result,
 351                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 352                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 353                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 354                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 355         )
 356
 357     def test_stdout(self):
 358         # 1. prepare the checklist
 359         config_checklist = []
 360         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 361                                 AND(CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 362                                     SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')))]
 363         config_checklist += [AND(CmdlineCheck('reason_4', 'decision_4', 'name_4', 'expected_4'),
 364                                  OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 365                                     SysctlCheck('reason_6', 'decision_6', 'name_6', 'expected_6')))]
 366
 367         # 2. prepare the parsed cmdline options
 368         parsed_cmdline_options = OrderedDict()
 369         parsed_cmdline_options['name_4'] = 'expected_4'
 370
 371         # 3. prepare the parsed sysctl options
 372         parsed_sysctl_options = OrderedDict()
 373         parsed_sysctl_options['name_3'] = 'UNexpected_3'
 374         parsed_sysctl_options['name_6'] = 'UNexpected_6'
 375
 376         # 4. run the engine
 377         self.run_engine(config_checklist, None, parsed_cmdline_options, parsed_sysctl_options, None)
 378
 379         # 5. check that the results are correct
 380         json_result = []
 381         self.get_engine_result(config_checklist, json_result, 'json')
 382         self.assertEqual(
 383                 json_result,
 384                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: is not found"],
 385                  ["name_4", "cmdline", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""]]
 386         )
 387
 388         stdout_result = []
 389         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 390         self.assertEqual(
 391                 stdout_result,
 392                 [
 393 "\
 394 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\
 395 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | FAIL: CONFIG_NAME_5 is not \"expected_5\"\
 396 "               ]
 397         )
 398
 399         stdout_result = []
 400         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 401         self.assertEqual(
 402                 stdout_result,
 403                 [
 404 "\
 405     <<< OR >>>                                                                             | FAIL: is not found\n\
 406 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\n\
 407     <<< AND >>>                                                                            | FAIL: name_3 is not \"expected_3\"\n\
 408 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | None\n\
 409 name_3                                  |sysctl | expected_3 |decision_3|     reason_3     | FAIL: \"UNexpected_3\"\
 410 "\
 411 "\
 412     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_5 is not \"expected_5\"\n\
 413 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | None\n\
 414     <<< OR >>>                                                                             | FAIL: is not found\n\
 415 CONFIG_NAME_5                           |kconfig| expected_5 |decision_5|     reason_5     | FAIL: is not found\n\
 416 name_6                                  |sysctl | expected_6 |decision_6|     reason_6     | FAIL: \"UNexpected_6\"\
 417 "               ]
 418         )
 419
 420     def test_value_overriding(self):
 421         # 1. prepare the checklist
 422         config_checklist = []
 423         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 424         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 425         config_checklist += [SysctlCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 426
 427         # 2. prepare the parsed kconfig options
 428         parsed_kconfig_options = OrderedDict()
 429         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 430
 431         # 3. prepare the parsed cmdline options
 432         parsed_cmdline_options = OrderedDict()
 433         parsed_cmdline_options['name_2'] = 'expected_2_new'
 434
 435         # 4. prepare the parsed sysctl options
 436         parsed_sysctl_options = OrderedDict()
 437         parsed_sysctl_options['name_3'] = 'expected_3_new'
 438
 439         # 5. run the engine
 440         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, parsed_sysctl_options, None)
 441
 442         # 6. check that the results are correct
 443         result = []
 444         self.get_engine_result(config_checklist, result, 'json')
 445         self.assertEqual(
 446                 result,
 447                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 448                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 449                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 450         )
 451
 452         # 7. override expected value and perform the checks again
 453         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 454         perform_checks(config_checklist)
 455
 456         # 8. check that the results are correct
 457         result = []
 458         self.get_engine_result(config_checklist, result, 'json')
 459         self.assertEqual(
 460                 result,
 461                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 462                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""],
 463                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 464         )
 465
 466         # 9. override expected value and perform the checks again
 467         override_expected_value(config_checklist, "name_2", "expected_2_new")
 468         perform_checks(config_checklist)
 469
 470         # 10. check that the results are correct
 471         result = []
 472         self.get_engine_result(config_checklist, result, 'json')
 473         self.assertEqual(
 474                 result,
 475                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 476                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 477                  ["name_3", "sysctl", "expected_3", "decision_3", "reason_3", "FAIL: \"expected_3_new\""]]
 478         )
 479
 480         # 11. override expected value and perform the checks again
 481         override_expected_value(config_checklist, "name_3", "expected_3_new")
 482         perform_checks(config_checklist)
 483
 484         # 12. check that the results are correct
 485         result = []
 486         self.get_engine_result(config_checklist, result, 'json')
 487         self.assertEqual(
 488                 result,
 489                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 490                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"],
 491                  ["name_3", "sysctl", "expected_3_new", "decision_3", "reason_3", "OK"]]
 492         )