test_engine: implement get_engine_result() for stdout
[kconfig-hardened-check.git] / kconfig_hardened_check / test_engine.py
index 473e2f7f13a8a4ea2e45296c52f2ec2403fca5fc..960f5168f8fa14f4c0996ccbb0260d29d2e508a1 100644 (file)
@@ -13,9 +13,11 @@ This module performs unit-testing of the kconfig-hardened-check engine.
 # pylint: disable=missing-function-docstring,line-too-long
 
 import unittest
+import io
+import sys
 from collections import OrderedDict
 import json
-from .engine import KconfigCheck, CmdlineCheck, OR, AND, populate_with_data, perform_checks
+from .engine import KconfigCheck, CmdlineCheck, VersionCheck, OR, AND, populate_with_data, perform_checks
 
 
 class TestEngine(unittest.TestCase):
@@ -39,19 +41,16 @@ class TestEngine(unittest.TestCase):
         kernel_version = (42, 43)
 
         # 5. run the engine
-        result = []
-        self.run_engine(config_checklist,
-                        parsed_kconfig_options, parsed_cmdline_options, kernel_version,
-                        result)
+        self.run_engine(config_checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version)
 
         # 6. check that the results are correct
-        # self.assertEqual(...
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
+        self.assertEqual(...
     """
 
     @staticmethod
-    def run_engine(checklist,
-                   parsed_kconfig_options, parsed_cmdline_options, kernel_version,
-                   result):
+    def run_engine(checklist, parsed_kconfig_options, parsed_cmdline_options, kernel_version):
         # populate the checklist with data
         if parsed_kconfig_options:
             populate_with_data(checklist, parsed_kconfig_options, 'kconfig')
@@ -66,16 +65,36 @@ class TestEngine(unittest.TestCase):
         # print the table with the results
         print('TABLE:')
         for opt in checklist:
-            opt.table_print(None, True) # default mode, with_results
+            opt.table_print('verbose', True) # verbose mode, with_results
             print()
+            print('=' * 121)
 
         # print the results in JSON
         print('JSON:')
+        result = []
         for opt in checklist:
             result.append(opt.json_dump(True)) # with_results
         print(json.dumps(result))
         print()
 
+    @staticmethod
+    def get_engine_result(checklist, result, result_type):
+        assert(result_type in ('json', 'stdout')), \
+               f'invalid result type "{result_type}"'
+
+        if result_type == 'json':
+            for opt in checklist:
+                result.append(opt.json_dump(True)) # with_results
+            return
+
+        captured_output = io.StringIO()
+        stdout_backup = sys.stdout
+        sys.stdout = captured_output
+        for opt in checklist:
+            opt.table_print('verbose', True) # verbose mode, with_results
+        sys.stdout = stdout_backup
+        result.append(captured_output.getvalue())
+
     def test_single_kconfig(self):
         # 1. prepare the checklist
         config_checklist = []
@@ -100,10 +119,11 @@ class TestEngine(unittest.TestCase):
         parsed_kconfig_options['CONFIG_NAME_9'] = '0'
 
         # 3. run the engine
-        result = []
-        self.run_engine(config_checklist, parsed_kconfig_options, None, None, result)
+        self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 
         # 4. check that the results are correct
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
         self.assertEqual(
                 result,
                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
@@ -142,10 +162,11 @@ class TestEngine(unittest.TestCase):
         parsed_cmdline_options['name_9'] = '0'
 
         # 3. run the engine
-        result = []
-        self.run_engine(config_checklist, None, parsed_cmdline_options, None, result)
+        self.run_engine(config_checklist, None, parsed_cmdline_options, None)
 
         # 4. check that the results are correct
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
         self.assertEqual(
                 result,
                 [["name_1", "cmdline", "expected_1", "decision_1", "reason_1", "OK"],
@@ -188,10 +209,11 @@ class TestEngine(unittest.TestCase):
         parsed_kconfig_options['CONFIG_NAME_11'] = 'really_not_off'
 
         # 3. run the engine
-        result = []
-        self.run_engine(config_checklist, parsed_kconfig_options, None, None, result)
+        self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 
         # 4. check that the results are correct
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
         self.assertEqual(
                 result,
                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
@@ -232,10 +254,11 @@ class TestEngine(unittest.TestCase):
         parsed_kconfig_options['CONFIG_NAME_12'] = 'expected_12'
 
         # 3. run the engine
-        result = []
-        self.run_engine(config_checklist, parsed_kconfig_options, None, None, result)
+        self.run_engine(config_checklist, parsed_kconfig_options, None, None)
 
         # 4. check that the results are correct
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
         self.assertEqual(
                 result,
                 [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK"],
@@ -245,3 +268,38 @@ class TestEngine(unittest.TestCase):
                  ["CONFIG_NAME_10", "kconfig", "expected_10", "decision_10", "reason_10", "FAIL: CONFIG_NAME_11 is off"],
                  ["CONFIG_NAME_12", "kconfig", "expected_12", "decision_12", "reason_12", "FAIL: CONFIG_NAME_13 is off, not found"]]
         )
+
+    def test_version(self):
+        # 1. prepare the checklist
+        config_checklist = []
+        config_checklist += [OR(KconfigCheck('reason_1', 'decision_1', 'NAME_1', 'expected_1'),
+                                VersionCheck((41, 101)))]
+        config_checklist += [AND(KconfigCheck('reason_2', 'decision_2', 'NAME_2', 'expected_2'),
+                                VersionCheck((44, 1)))]
+        config_checklist += [AND(KconfigCheck('reason_3', 'decision_3', 'NAME_3', 'expected_3'),
+                                VersionCheck((42, 44)))]
+        config_checklist += [OR(KconfigCheck('reason_4', 'decision_4', 'NAME_4', 'expected_4'),
+                                VersionCheck((42, 43)))]
+
+        # 2. prepare the parsed kconfig options
+        parsed_kconfig_options = OrderedDict()
+        parsed_kconfig_options['CONFIG_NAME_2'] = 'expected_2'
+        parsed_kconfig_options['CONFIG_NAME_3'] = 'expected_3'
+
+        # 3. prepare the kernel version
+        kernel_version = (42, 43)
+
+        # 4. run the engine
+        self.run_engine(config_checklist, parsed_kconfig_options, None, kernel_version)
+
+        # 5. check that the results are correct
+        result = []
+        self.get_engine_result(config_checklist, result, 'json')
+        self.assertEqual(
+                result,
+                [["CONFIG_NAME_1", "kconfig", "expected_1", "decision_1", "reason_1", "OK: version >= 41.101"],
+                 ["CONFIG_NAME_2", "kconfig", "expected_2", "decision_2", "reason_2", "FAIL: version < 44.1"],
+                 ["CONFIG_NAME_3", "kconfig", "expected_3", "decision_3", "reason_3", "FAIL: version < 42.44"],
+                 ["CONFIG_NAME_4", "kconfig", "expected_4", "decision_4", "reason_4", "OK: version >= 42.43"]]
+        )
+