kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool helps me to check Linux kernel options against
   5 my security hardening preferences for X86_64, ARM64, X86_32, and ARM.
   6 Let the computers do their job!
   7
   8 Author: Alexander Popov <alex.popov@linux.com>
   9
  10 This module performs unit-testing of the kconfig-hardened-check engine.
  11 """
  12
  13 # pylint: disable=missing-function-docstring,line-too-long
  14
  15 import unittest
  16 import io
  17 import sys
  18 from collections import OrderedDict
  19 import json
  20 from .engine import KconfigCheck, CmdlineCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
  21
  22
  23 class TestEngine(unittest.TestCase):
  24     """
  25     Example test scenario:
  26
  27         # 1. prepare the checklist
  28         config_checklist = []
  29         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  30         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  31
  32         # 2. prepare the parsed kconfig options
  33         parsed_kconfig_options = OrderedDict()
  34         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  35
  36         # 3. prepare the parsed cmdline options
  37         parsed_cmdline_options = OrderedDict()
  38         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  39
  40         # 4. prepare the kernel version
  41         kernel_version = (42, 43)
  42
  43         # 5. run the engine
  44         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version)
  45
  46         # 6. check that the results are correct
  47         result = []
  48         self.get_engine_result(config_checklist, result, 'json')
  49         self.assertEqual(...
  50     """
  51
  52     @staticmethod
  53     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version):
  54         # populate the checklist with data
  55         if parsed_kconfig_options:
  56             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  57         if parsed_cmdline_options:
  58             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  59         if kernel_version:
  60             populate_with_data(checklist, kernel_version, 'version')
  61
  62         # now everything is ready, perform the checks
  63         perform_checks(checklist)
  64
  65         # print the table with the results
  66         print('TABLE:')
  67         for opt in checklist:
  68             opt.table_print('verbose', True) # verbose mode, with_results
  69             print()
  70             print('=' * 121)
  71
  72         # print the results in JSON
  73         print('JSON:')
  74         result = []
  75         for opt in checklist:
  76             result.append(opt.json_dump(True)) # with_results
  77         print(json.dumps(result))
  78         print()
  79
  80     @staticmethod
  81     def get_engine_result(checklist, result, result_type):
  82         assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
  83                f'invalid result type "{result_type}"'
  84
  85         if result_type == 'json':
  86             for opt in checklist:
  87                 result.append(opt.json_dump(True)) # with_results
  88             return
  89
  90         captured_output = io.StringIO()
  91         stdout_backup = sys.stdout
  92         sys.stdout = captured_output
  93         for opt in checklist:
  94             if result_type == 'stdout_verbose':
  95                 opt.table_print('verbose', True) # verbose mode, with_results
  96             else:
  97                 opt.table_print(None, True) # normal mode, with_results
  98         sys.stdout = stdout_backup
  99         result.append(captured_output.getvalue())
 100
 101     def test_simple_kconfig(self):
 102         # 1. prepare the checklist
 103         config_checklist = []
 104         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 105         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 106         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 107         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 108         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 109         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 110         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 111         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 112         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 113         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 114
 115         # 2. prepare the parsed kconfig options
 116         parsed_kconfig_options = OrderedDict()
 117         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 118         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 119         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 120         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 121         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 122         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 123
 124         # 3. run the engine
 125         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 126
 127         # 4. check that the results are correct
 128         result = []
 129         self.get_engine_result(config_checklist, result, 'json')
 130         self.assertEqual(
 131                 result,
 132                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 133                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 134                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 135                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 136                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 137                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 138                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 139                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 140                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 141                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 142         )
 143
 144     def test_simple_cmdline(self):
 145         # 1. prepare the checklist
 146         config_checklist = []
 147         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 148         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 149         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 150         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 151         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 152         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 153         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 154         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 155         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 156         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 157
 158         # 2. prepare the parsed cmdline options
 159         parsed_cmdline_options = OrderedDict()
 160         parsed_cmdline_options['name_1'] = 'expected_1'
 161         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 162         parsed_cmdline_options['name_5'] = ''
 163         parsed_cmdline_options['name_7'] = ''
 164         parsed_cmdline_options['name_8'] = 'off'
 165         parsed_cmdline_options['name_9'] = '0'
 166
 167         # 3. run the engine
 168         self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 169
 170         # 4. check that the results are correct
 171         result = []
 172         self.get_engine_result(config_checklist, result, 'json')
 173         self.assertEqual(
 174                 result,
 175                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 176                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 177                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 178                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 179                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 180                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 181                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 182                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 183                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 184                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 185         )
 186
 187     def test_complex_or(self):
 188         # 1. prepare the checklist
 189         config_checklist = []
 190         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 191                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 192         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 193                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 194         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 195                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 196         config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
 197                                 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
 198         config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 199                                 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 200         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 201                                 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 202
 203         # 2. prepare the parsed kconfig options
 204         parsed_kconfig_options = OrderedDict()
 205         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 206         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 207         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 208         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 209         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 210         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 211         parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
 212         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 213
 214         # 3. run the engine
 215         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 216
 217         # 4. check that the results are correct
 218         result = []
 219         self.get_engine_result(config_checklist, result, 'json')
 220         self.assertEqual(
 221                 result,
 222                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 223                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 224                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 225                  ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
 226                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
 227                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
 228         )
 229
 230     def test_complex_and(self):
 231         # 1. prepare the checklist
 232         config_checklist = []
 233         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 234                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 235         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 236                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 237         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 238                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 239         config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 240                                  KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 241         config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 242                                  KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 243         config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
 244                                  KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
 245
 246         # 2. prepare the parsed kconfig options
 247         parsed_kconfig_options = OrderedDict()
 248         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 249         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 250         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 251         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 252         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 253         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 254         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 255         parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
 256         parsed_kconfig_options['CONFIG_NAME_11'] = '0'
 257         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 258
 259         # 3. run the engine
 260         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 261
 262         # 4. check that the results are correct
 263         result = []
 264         self.get_engine_result(config_checklist, result, 'json')
 265         self.assertEqual(
 266                 result,
 267                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 268                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 269                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 270                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
 271                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
 272                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
 273         )
 274
 275     def test_version(self):
 276         # 1. prepare the checklist
 277         config_checklist = []
 278         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 279                                 VersionCheck((41, 101)))]
 280         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 281                                 VersionCheck((44, 1)))]
 282         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 283                                 VersionCheck((42, 44)))]
 284         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 285                                 VersionCheck((42, 43)))]
 286
 287         # 2. prepare the parsed kconfig options
 288         parsed_kconfig_options = OrderedDict()
 289         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 290         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 291
 292         # 3. prepare the kernel version
 293         kernel_version = (42, 43)
 294
 295         # 4. run the engine
 296         self.run_engine(config_checklist, parsed_kconfig_options, None, kernel_version)
 297
 298         # 5. check that the results are correct
 299         result = []
 300         self.get_engine_result(config_checklist, result, 'json')
 301         self.assertEqual(
 302                 result,
 303                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 304                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 305                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 306                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 307         )
 308
 309     def test_stdout(self):
 310         # 1. prepare the checklist
 311         config_checklist = []
 312         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 313                                 AND(CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
 314                                     KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')))]
 315         config_checklist += [AND(CmdlineCheck('reason_4', 'decision_4', 'name_4', 'expected_4'),
 316                                  OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 317                                     CmdlineCheck('reason_6', 'decision_6', 'name_6', 'expected_6')))]
 318
 319         # 2. prepare the parsed cmdline options
 320         parsed_cmdline_options = OrderedDict()
 321         parsed_cmdline_options['name_4'] = 'expected_4'
 322         parsed_cmdline_options['name_6'] = 'UNexpected_6'
 323
 324         # 3. run the engine
 325         self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 326
 327         # 4. check that the results are correct
 328         json_result = []
 329         self.get_engine_result(config_checklist, json_result, 'json')
 330         self.assertEqual(
 331                 json_result,
 332                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: is not found"],
 333                  ["name_4", "cmdline", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""]]
 334         )
 335
 336         stdout_result = []
 337         self.get_engine_result(config_checklist, stdout_result, 'stdout')
 338         self.assertEqual(
 339                 stdout_result,
 340                 [
 341 "\
 342 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\
 343 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | FAIL: CONFIG_NAME_5 is not \"expected_5\"\
 344 "               ]
 345         )
 346
 347         stdout_result = []
 348         self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
 349         self.assertEqual(
 350                 stdout_result,
 351                 [
 352 "\
 353     <<< OR >>>                                                                             | FAIL: is not found\n\
 354 CONFIG_NAME_1                           |kconfig| expected_1 |decision_1|     reason_1     | FAIL: is not found\n\
 355     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_3 is not \"expected_3\"\n\
 356 name_2                                  |cmdline| expected_2 |decision_2|     reason_2     | None\n\
 357 CONFIG_NAME_3                           |kconfig| expected_3 |decision_3|     reason_3     | FAIL: is not found\
 358 "\
 359 "\
 360     <<< AND >>>                                                                            | FAIL: CONFIG_NAME_5 is not \"expected_5\"\n\
 361 name_4                                  |cmdline| expected_4 |decision_4|     reason_4     | None\n\
 362     <<< OR >>>                                                                             | FAIL: is not found\n\
 363 CONFIG_NAME_5                           |kconfig| expected_5 |decision_5|     reason_5     | FAIL: is not found\n\
 364 name_6                                  |cmdline| expected_6 |decision_6|     reason_6     | FAIL: \"UNexpected_6\"\
 365 "               ]
 366         )
 367
 368     def test_value_overriding(self):
 369         # 1. prepare the checklist
 370         config_checklist = []
 371         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 372         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 373
 374         # 2. prepare the parsed kconfig options
 375         parsed_kconfig_options = OrderedDict()
 376         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
 377
 378         # 3. prepare the parsed cmdline options
 379         parsed_cmdline_options = OrderedDict()
 380         parsed_cmdline_options['name_2'] = 'expected_2_new'
 381
 382         # 4. run the engine
 383         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, None)
 384
 385         # 5. check that the results are correct
 386         result = []
 387         self.get_engine_result(config_checklist, result, 'json')
 388         self.assertEqual(
 389                 result,
 390                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
 391                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
 392         )
 393
 394         # 6. override expected value and perform the checks again
 395         override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
 396         perform_checks(config_checklist)
 397
 398         # 7. check that the results are correct
 399         result = []
 400         self.get_engine_result(config_checklist, result, 'json')
 401         self.assertEqual(
 402                 result,
 403                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 404                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
 405         )
 406
 407         # 8. override expected value and perform the checks again
 408         override_expected_value(config_checklist, "name_2", "expected_2_new")
 409         perform_checks(config_checklist)
 410
 411         # 9. check that the results are correct
 412         result = []
 413         self.get_engine_result(config_checklist, result, 'json')
 414         self.assertEqual(
 415                 result,
 416                 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
 417                  ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"]]
 418         )