4 This tool is for checking the security hardening options of the Linux kernel.
6 Author: Alexander Popov <alex.popov@linux.com>
8 This module performs unit-testing of the kconfig-hardened-check engine.
11 # pylint: disable=missing-function-docstring,line-too-long
16 from collections import OrderedDict
18 from .engine import KconfigCheck, CmdlineCheck, VersionCheck, OR, AND, populate_with_data, perform_checks, override_expected_value
21 class TestEngine(unittest.TestCase):
23 Example test scenario:
25 # 1. prepare the checklist
27 config_checklist += [KconfigCheck('reason_1', 'decision_1', 'KCONFIG_NAME', 'expected_1')]
28 config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'cmdline_name', 'expected_2')]
30 # 2. prepare the parsed kconfig options
31 parsed_kconfig_options = OrderedDict()
32 parsed_kconfig_options['CONFIG_KCONFIG_NAME'] = 'UNexpected_1'
34 # 3. prepare the parsed cmdline options
35 parsed_cmdline_options = OrderedDict()
36 parsed_cmdline_options['cmdline_name'] = 'expected_2'
38 # 4. prepare the kernel version
39 kernel_version = (42, 43)
42 self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version)
44 # 6. check that the results are correct
46 self.get_engine_result(config_checklist, result, 'json')
51 def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version):
52 # populate the checklist with data
53 if parsed_kconfig_options:
54 populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
55 if parsed_cmdline_options:
56 populate_with_data(checklist, parsed_cmdline_options, 'cmdline')
58 populate_with_data(checklist, kernel_version, 'version')
60 # now everything is ready, perform the checks
61 perform_checks(checklist)
63 # print the table with the results
66 opt.table_print('verbose', True) # verbose mode, with_results
70 # print the results in JSON
74 result.append(opt.json_dump(True)) # with_results
75 print(json.dumps(result))
79 def get_engine_result(checklist, result, result_type):
80 assert(result_type in ('json', 'stdout', 'stdout_verbose')), \
81 f'invalid result type "{result_type}"'
83 if result_type == 'json':
85 result.append(opt.json_dump(True)) # with_results
88 captured_output = io.StringIO()
89 stdout_backup = sys.stdout
90 sys.stdout = captured_output
92 if result_type == 'stdout_verbose':
93 opt.table_print('verbose', True) # verbose mode, with_results
95 opt.table_print(None, True) # normal mode, with_results
96 sys.stdout = stdout_backup
97 result.append(captured_output.getvalue())
99 def test_simple_kconfig(self):
100 # 1. prepare the checklist
101 config_checklist = []
102 config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
103 config_checklist += [KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2')]
104 config_checklist += [KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')]
105 config_checklist += [KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'is not set')]
106 config_checklist += [KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'is present')]
107 config_checklist += [KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'is present')]
108 config_checklist += [KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not off')]
109 config_checklist += [KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'is not off')]
110 config_checklist += [KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is not off')]
111 config_checklist += [KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'is not off')]
113 # 2. prepare the parsed kconfig options
114 parsed_kconfig_options = OrderedDict()
115 parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
116 parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
117 parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
118 parsed_kconfig_options['CONFIG_NAME_7'] = 'really_not_off'
119 parsed_kconfig_options['CONFIG_NAME_8'] = 'off'
120 parsed_kconfig_options['CONFIG_NAME_9'] = '0'
123 self.run_engine(config_checklist, parsed_kconfig_options, None, None)
125 # 4. check that the results are correct
127 self.get_engine_result(config_checklist, result, 'json')
130 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
131 ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
132 ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
133 ["CONFIG_NAME_4", "kconfig", "is not set", "decision_4", "reason_4", "OK: is not found"],
134 ["CONFIG_NAME_5", "kconfig", "is present", "decision_5", "reason_5", "OK: is present"],
135 ["CONFIG_NAME_6", "kconfig", "is present", "decision_6", "reason_6", "FAIL: is not present"],
136 ["CONFIG_NAME_7", "kconfig", "is not off", "decision_7", "reason_7", "OK: is not off, \"really_not_off\""],
137 ["CONFIG_NAME_8", "kconfig", "is not off", "decision_8", "reason_8", "FAIL: is off"],
138 ["CONFIG_NAME_9", "kconfig", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
139 ["CONFIG_NAME_10", "kconfig", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
142 def test_simple_cmdline(self):
143 # 1. prepare the checklist
144 config_checklist = []
145 config_checklist += [CmdlineCheck('reason_1', 'decision_1', 'name_1', 'expected_1')]
146 config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
147 config_checklist += [CmdlineCheck('reason_3', 'decision_3', 'name_3', 'expected_3')]
148 config_checklist += [CmdlineCheck('reason_4', 'decision_4', 'name_4', 'is not set')]
149 config_checklist += [CmdlineCheck('reason_5', 'decision_5', 'name_5', 'is present')]
150 config_checklist += [CmdlineCheck('reason_6', 'decision_6', 'name_6', 'is present')]
151 config_checklist += [CmdlineCheck('reason_7', 'decision_7', 'name_7', 'is not off')]
152 config_checklist += [CmdlineCheck('reason_8', 'decision_8', 'name_8', 'is not off')]
153 config_checklist += [CmdlineCheck('reason_9', 'decision_9', 'name_9', 'is not off')]
154 config_checklist += [CmdlineCheck('reason_10', 'decision_10', 'name_10', 'is not off')]
156 # 2. prepare the parsed cmdline options
157 parsed_cmdline_options = OrderedDict()
158 parsed_cmdline_options['name_1'] = 'expected_1'
159 parsed_cmdline_options['name_2'] = 'UNexpected_2'
160 parsed_cmdline_options['name_5'] = ''
161 parsed_cmdline_options['name_7'] = ''
162 parsed_cmdline_options['name_8'] = 'off'
163 parsed_cmdline_options['name_9'] = '0'
166 self.run_engine(config_checklist, None, parsed_cmdline_options, None)
168 # 4. check that the results are correct
170 self.get_engine_result(config_checklist, result, 'json')
173 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
174 ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"UNexpected_2\""],
175 ["name_3", "cmdline", "expected_3", "decision_3", "reason_3", "FAIL: is not found"],
176 ["name_4", "cmdline", "is not set", "decision_4", "reason_4", "OK: is not found"],
177 ["name_5", "cmdline", "is present", "decision_5", "reason_5", "OK: is present"],
178 ["name_6", "cmdline", "is present", "decision_6", "reason_6", "FAIL: is not present"],
179 ["name_7", "cmdline", "is not off", "decision_7", "reason_7", "OK: is not off, \"\""],
180 ["name_8", "cmdline", "is not off", "decision_8", "reason_8", "FAIL: is off"],
181 ["name_9", "cmdline", "is not off", "decision_9", "reason_9", "FAIL: is off, \"0\""],
182 ["name_10", "cmdline", "is not off", "decision_10", "reason_10", "FAIL: is off, not found"]]
185 def test_complex_or(self):
186 # 1. prepare the checklist
187 config_checklist = []
188 config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
189 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
190 config_checklist += [OR(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
191 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
192 config_checklist += [OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
193 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
194 config_checklist += [OR(KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'),
195 KconfigCheck('reason_7', 'decision_7', 'NAME_7', 'is not set'))]
196 config_checklist += [OR(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
197 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
198 config_checklist += [OR(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
199 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
201 # 2. prepare the parsed kconfig options
202 parsed_kconfig_options = OrderedDict()
203 parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
204 parsed_kconfig_options['CONFIG_NAME_2'] = 'UNexpected_2'
205 parsed_kconfig_options['CONFIG_NAME_3'] = 'UNexpected_3'
206 parsed_kconfig_options['CONFIG_NAME_4'] = 'expected_4'
207 parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
208 parsed_kconfig_options['CONFIG_NAME_6'] = 'UNexpected_6'
209 parsed_kconfig_options['CONFIG_NAME_9'] = 'UNexpected_9'
210 parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
213 self.run_engine(config_checklist, parsed_kconfig_options, None, None)
215 # 4. check that the results are correct
217 self.get_engine_result(config_checklist, result, 'json')
220 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
221 ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "OK: CONFIG_NAME_4 is \"expected_4\""],
222 ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
223 ["CONFIG_NAME_6", "kconfig", "expected_6", "decision_6", "reason_6", "OK: CONFIG_NAME_7 is not found"],
224 ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "OK: CONFIG_NAME_9 is present"],
225 ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "OK: CONFIG_NAME_11 is not off"]]
228 def test_complex_and(self):
229 # 1. prepare the checklist
230 config_checklist = []
231 config_checklist += [AND(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
232 KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'))]
233 config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
234 KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'))]
235 config_checklist += [AND(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
236 KconfigCheck('reason_6', 'decision_6', 'NAME_6', 'expected_6'))]
237 config_checklist += [AND(KconfigCheck('reason_8', 'decision_8', 'NAME_8', 'expected_8'),
238 KconfigCheck('reason_9', 'decision_9', 'NAME_9', 'is present'))]
239 config_checklist += [AND(KconfigCheck('reason_10', 'decision_10', 'NAME_10', 'expected_10'),
240 KconfigCheck('reason_11', 'decision_11', 'NAME_11', 'is not off'))]
241 config_checklist += [AND(KconfigCheck('reason_12', 'decision_12', 'NAME_12', 'expected_12'),
242 KconfigCheck('reason_13', 'decision_13', 'NAME_13', 'is not off'))]
244 # 2. prepare the parsed kconfig options
245 parsed_kconfig_options = OrderedDict()
246 parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1'
247 parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
248 parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
249 parsed_kconfig_options['CONFIG_NAME_4'] = 'UNexpected_4'
250 parsed_kconfig_options['CONFIG_NAME_5'] = 'UNexpected_5'
251 parsed_kconfig_options['CONFIG_NAME_6'] = 'expected_6'
252 parsed_kconfig_options['CONFIG_NAME_8'] = 'expected_8'
253 parsed_kconfig_options['CONFIG_NAME_10'] = 'expected_10'
254 parsed_kconfig_options['CONFIG_NAME_11'] = '0'
255 parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
258 self.run_engine(config_checklist, parsed_kconfig_options, None, None)
260 # 4. check that the results are correct
262 self.get_engine_result(config_checklist, result, 'json')
265 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
266 ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: CONFIG_NAME_4 is not \"expected_4\""],
267 ["CONFIG_NAME_5", "kconfig", "expected_5", "decision_5", "reason_5", "FAIL: \"UNexpected_5\""],
268 ["CONFIG_NAME_8", "kconfig", "expected_8", "decision_8", "reason_8", "FAIL: CONFIG_NAME_9 is not present"],
269 ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
270 ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
273 def test_version(self):
274 # 1. prepare the checklist
275 config_checklist = []
276 config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
277 VersionCheck((41, 101)))]
278 config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
279 VersionCheck((44, 1)))]
280 config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
281 VersionCheck((42, 44)))]
282 config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
283 VersionCheck((42, 43)))]
285 # 2. prepare the parsed kconfig options
286 parsed_kconfig_options = OrderedDict()
287 parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
288 parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
290 # 3. prepare the kernel version
291 kernel_version = (42, 43)
294 self.run_engine(config_checklist, parsed_kconfig_options, None, kernel_version)
296 # 5. check that the results are correct
298 self.get_engine_result(config_checklist, result, 'json')
301 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
302 ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
303 ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
304 ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
307 def test_stdout(self):
308 # 1. prepare the checklist
309 config_checklist = []
310 config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
311 AND(CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2'),
312 KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3')))]
313 config_checklist += [AND(CmdlineCheck('reason_4', 'decision_4', 'name_4', 'expected_4'),
314 OR(KconfigCheck('reason_5', 'decision_5', 'NAME_5', 'expected_5'),
315 CmdlineCheck('reason_6', 'decision_6', 'name_6', 'expected_6')))]
317 # 2. prepare the parsed cmdline options
318 parsed_cmdline_options = OrderedDict()
319 parsed_cmdline_options['name_4'] = 'expected_4'
320 parsed_cmdline_options['name_6'] = 'UNexpected_6'
323 self.run_engine(config_checklist, None, parsed_cmdline_options, None)
325 # 4. check that the results are correct
327 self.get_engine_result(config_checklist, json_result, 'json')
330 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: is not found"],
331 ["name_4", "cmdline", "expected_4", "decision_4", "reason_4", "FAIL: CONFIG_NAME_5 is not \"expected_5\""]]
335 self.get_engine_result(config_checklist, stdout_result, 'stdout')
340 CONFIG_NAME_1 |kconfig| expected_1 |decision_1| reason_1 | FAIL: is not found\
341 name_4 |cmdline| expected_4 |decision_4| reason_4 | FAIL: CONFIG_NAME_5 is not \"expected_5\"\
346 self.get_engine_result(config_checklist, stdout_result, 'stdout_verbose')
351 <<< OR >>> | FAIL: is not found\n\
352 CONFIG_NAME_1 |kconfig| expected_1 |decision_1| reason_1 | FAIL: is not found\n\
353 <<< AND >>> | FAIL: CONFIG_NAME_3 is not \"expected_3\"\n\
354 name_2 |cmdline| expected_2 |decision_2| reason_2 | None\n\
355 CONFIG_NAME_3 |kconfig| expected_3 |decision_3| reason_3 | FAIL: is not found\
358 <<< AND >>> | FAIL: CONFIG_NAME_5 is not \"expected_5\"\n\
359 name_4 |cmdline| expected_4 |decision_4| reason_4 | None\n\
360 <<< OR >>> | FAIL: is not found\n\
361 CONFIG_NAME_5 |kconfig| expected_5 |decision_5| reason_5 | FAIL: is not found\n\
362 name_6 |cmdline| expected_6 |decision_6| reason_6 | FAIL: \"UNexpected_6\"\
366 def test_value_overriding(self):
367 # 1. prepare the checklist
368 config_checklist = []
369 config_checklist += [KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1')]
370 config_checklist += [CmdlineCheck('reason_2', 'decision_2', 'name_2', 'expected_2')]
372 # 2. prepare the parsed kconfig options
373 parsed_kconfig_options = OrderedDict()
374 parsed_kconfig_options['CONFIG_NAME_1'] = 'expected_1_new'
376 # 3. prepare the parsed cmdline options
377 parsed_cmdline_options = OrderedDict()
378 parsed_cmdline_options['name_2'] = 'expected_2_new'
381 self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, None)
383 # 5. check that the results are correct
385 self.get_engine_result(config_checklist, result, 'json')
388 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "FAIL: \"expected_1_new\""],
389 ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
392 # 6. override expected value and perform the checks again
393 override_expected_value(config_checklist, "CONFIG_NAME_1", "expected_1_new")
394 perform_checks(config_checklist)
396 # 7. check that the results are correct
398 self.get_engine_result(config_checklist, result, 'json')
401 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
402 ["name_2", "cmdline", "expected_2", "decision_2", "reason_2", "FAIL: \"expected_2_new\""]]
405 # 8. override expected value and perform the checks again
406 override_expected_value(config_checklist, "name_2", "expected_2_new")
407 perform_checks(config_checklist)
409 # 9. check that the results are correct
411 self.get_engine_result(config_checklist, result, 'json')
414 [["CONFIG_NAME_1", "kconfig", "expected_1_new", "decision_1", "reason_1", "OK"],
415 ["name_2", "cmdline", "expected_2_new", "decision_2", "reason_2", "OK"]]