kconfig_hardened_check/test_engine.py

   1 #!/usr/bin/python3
   2
   3 """
   4 This tool helps me to check Linux kernel options against
   5 my security hardening preferences for X86_64, ARM64, X86_32, and ARM.
   6 Let the computers do their job!
   7
   8 Author: Alexander Popov <alex.popov@linux.com>
   9
  10 This module performs unit-testing of the kconfig-hardened-check engine.
  11 """
  12
  13 # pylint: disable=missing-function-docstring,line-too-long
  14
  15 import unittest
  16 import io
  17 import sys
  18 from collections import OrderedDict
  19 import json
  20 from .engine import KconfigCheck, CmdlineCheck, VersionCheck, OR, AND, populate_with_data, perform_checks
  21
  22
  23 class TestEngine(unittest.TestCase):
  24     """
  25     Example test scenario:
  26
  27         # 1. prepare the checklist
  28         config_checklist = []
  29         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
  30         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
  31
  32         # 2. prepare the parsed kconfig options
  33         parsed_kconfig_options = OrderedDict()
  34         parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
  35
  36         # 3. prepare the parsed cmdline options
  37         parsed_cmdline_options = OrderedDict()
  38         parsed_cmdline_options['cmdline_name'] = 'expected_2'
  39
  40         # 4. prepare the kernel version
  41         kernel_version = (42, 43)
  42
  43         # 5. run the engine
  44         self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version)
  45
  46         # 6. check that the results are correct
  47         result = []
  48         self.get_engine_result(config_checklist, result, 'json')
  49         self.assertEqual(...
  50     """
  51
  52     @staticmethod
  53     def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version):
  54         # populate the checklist with data
  55         if parsed_kconfig_options:
  56             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
  57         if parsed_cmdline_options:
  58             populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
  59         if kernel_version:
  60             populate_with_data(checklist, kernel_version, 'version')
  61
  62         # now everything is ready, perform the checks
  63         perform_checks(checklist)
  64
  65         # print the table with the results
  66         print('TABLE:')
  67         for opt in checklist:
  68             opt.table_print('verbose', True) # verbose mode, with_results
  69             print()
  70             print('=' * 121)
  71
  72         # print the results in JSON
  73         print('JSON:')
  74         result = []
  75         for opt in checklist:
  76             result.append(opt.json_dump(True)) # with_results
  77         print(json.dumps(result))
  78         print()
  79
  80     @staticmethod
  81     def get_engine_result(checklist, result, result_type):
  82         assert(result_type in ('json', 'stdout')), \
  83                f'invalid result type "{result_type}"'
  84
  85         if result_type == 'json':
  86             for opt in checklist:
  87                 result.append(opt.json_dump(True)) # with_results
  88             return
  89
  90         captured_output = io.StringIO()
  91         stdout_backup = sys.stdout
  92         sys.stdout = captured_output
  93         for opt in checklist:
  94             opt.table_print('verbose', True) # verbose mode, with_results
  95         sys.stdout = stdout_backup
  96         result.append(captured_output.getvalue())
  97
  98     def test_single_kconfig(self):
  99         # 1. prepare the checklist
 100         config_checklist = []
 101         config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
 102         config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
 103         config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
 104         config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
 105         config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
 106         config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
 107         config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
 108         config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
 109         config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
 110         config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
 111
 112         # 2. prepare the parsed kconfig options
 113         parsed_kconfig_options = OrderedDict()
 114         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 115         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 116         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 117         parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
 118         parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
 119         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 120
 121         # 3. run the engine
 122         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 123
 124         # 4. check that the results are correct
 125         result = []
 126         self.get_engine_result(config_checklist, result, 'json')
 127         self.assertEqual(
 128                 result,
 129                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 130                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 131                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 132                  ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
 133                  ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
 134                  ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 135                  ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
 136                  ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 137                  ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 138                  ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 139         )
 140
 141     def test_single_cmdline(self):
 142         # 1. prepare the checklist
 143         config_checklist = []
 144         config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
 145         config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
 146         config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
 147         config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
 148         config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
 149         config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
 150         config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
 151         config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
 152         config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
 153         config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
 154
 155         # 2. prepare the parsed cmdline options
 156         parsed_cmdline_options = OrderedDict()
 157         parsed_cmdline_options['name_1'] = 'expected_1'
 158         parsed_cmdline_options['name_2'] = 'UNexpected_2'
 159         parsed_cmdline_options['name_5'] = ''
 160         parsed_cmdline_options['name_7'] = ''
 161         parsed_cmdline_options['name_8'] = 'off'
 162         parsed_cmdline_options['name_9'] = '0'
 163
 164         # 3. run the engine
 165         self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 166
 167         # 4. check that the results are correct
 168         result = []
 169         self.get_engine_result(config_checklist, result, 'json')
 170         self.assertEqual(
 171                 result,
 172                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
 173                  ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
 174                  ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
 175                  ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
 176                  ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
 177                  ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
 178                  ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
 179                  ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
 180                  ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
 181                  ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
 182         )
 183
 184     def test_OR(self):
 185         # 1. prepare the checklist
 186         config_checklist = []
 187         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 188                                 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 189         config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 190                                 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 191         config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 192                                 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 193         config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
 194                                 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
 195         config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 196                                 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 197         config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 198                                 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 199
 200         # 2. prepare the parsed kconfig options
 201         parsed_kconfig_options = OrderedDict()
 202         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 203         parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
 204         parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
 205         parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
 206         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 207         parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
 208         parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
 209         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 210
 211         # 3. run the engine
 212         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 213
 214         # 4. check that the results are correct
 215         result = []
 216         self.get_engine_result(config_checklist, result, 'json')
 217         self.assertEqual(
 218                 result,
 219                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 220                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
 221                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 222                  ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
 223                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
 224                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
 225         )
 226
 227     def test_AND(self):
 228         # 1. prepare the checklist
 229         config_checklist = []
 230         config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 231                                  KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
 232         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 233                                  KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
 234         config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
 235                                  KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
 236         config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
 237                                  KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
 238         config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
 239                                  KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
 240         config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
 241                                  KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
 242
 243         # 2. prepare the parsed kconfig options
 244         parsed_kconfig_options = OrderedDict()
 245         parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
 246         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 247         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 248         parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
 249         parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
 250         parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
 251         parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
 252         parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
 253         parsed_kconfig_options['CONFIG_NAME_11'] = '0'
 254         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 255
 256         # 3. run the engine
 257         self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 258
 259         # 4. check that the results are correct
 260         result = []
 261         self.get_engine_result(config_checklist, result, 'json')
 262         self.assertEqual(
 263                 result,
 264                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
 265                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
 266                  ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
 267                  ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
 268                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
 269                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
 270         )
 271
 272     def test_version(self):
 273         # 1. prepare the checklist
 274         config_checklist = []
 275         config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
 276                                 VersionCheck((41, 101)))]
 277         config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
 278                                 VersionCheck((44, 1)))]
 279         config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
 280                                 VersionCheck((42, 44)))]
 281         config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
 282                                 VersionCheck((42, 43)))]
 283
 284         # 2. prepare the parsed kconfig options
 285         parsed_kconfig_options = OrderedDict()
 286         parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
 287         parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
 288
 289         # 3. prepare the kernel version
 290         kernel_version = (42, 43)
 291
 292         # 4. run the engine
 293         self.run_engine(config_checklist, parsed_kconfig_options, None, kernel_version)
 294
 295         # 5. check that the results are correct
 296         result = []
 297         self.get_engine_result(config_checklist, result, 'json')
 298         self.assertEqual(
 299                 result,
 300                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
 301                  ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
 302                  ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
 303                  ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
 304         )
 305