From 2136dcfadd96f386bc8432c963065e274b382853 Mon Sep 17 00:00:00 2001 From: Alexander Popov Date: Mon, 8 May 2023 20:33:44 +0300 Subject: [PATCH] Run the functional tests and collect the coverage in Woodpecker-CI --- .github/workflows/functional_test.sh | 108 +++++++++++++++++++++++++++ .woodpecker/functional_test.yml | 12 ++- 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/functional_test.sh diff --git a/.github/workflows/functional_test.sh b/.github/workflows/functional_test.sh new file mode 100644 index 0000000..d9b9b37 --- /dev/null +++ b/.github/workflows/functional_test.sh @@ -0,0 +1,108 @@ +#!/bin/sh + +set -x +set -e + +echo "Beginning of the functional tests" + +echo ">>>>> get help <<<<<" +coverage run -a --branch bin/kconfig-hardened-check +coverage run -a --branch bin/kconfig-hardened-check -h + +echo ">>>>> get version <<<<<" +coverage run -a --branch bin/kconfig-hardened-check --version + +echo ">>>>> print the security hardening preferences <<<<<" +coverage run -a --branch bin/kconfig-hardened-check -p X86_64 +coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m verbose +coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m json + +coverage run -a --branch bin/kconfig-hardened-check -p X86_32 +coverage run -a --branch bin/kconfig-hardened-check -p X86_32 -m verbose +coverage run -a --branch bin/kconfig-hardened-check -p X86_32 -m json + +coverage run -a --branch bin/kconfig-hardened-check -p ARM64 +coverage run -a --branch bin/kconfig-hardened-check -p ARM64 -m verbose +coverage run -a --branch bin/kconfig-hardened-check -p ARM64 -m json + +coverage run -a --branch bin/kconfig-hardened-check -p ARM +coverage run -a --branch bin/kconfig-hardened-check -p ARM -m verbose +coverage run -a --branch bin/kconfig-hardened-check -p ARM -m json + +echo ">>>>> check the example kconfig files and cmdline <<<<<" +cat /proc/cmdline +echo "l1tf=off mds=full randomize_kstack_offset=on iommu.passthrough=0" > ./cmdline_example +cat ./cmdline_example +CONFIG_DIR=`find . -name config_files` +KCONFIGS=`find $CONFIG_DIR -type f | grep -e "\.config" -e "\.gz"` +COUNT=0 +for C in $KCONFIGS +do + COUNT=$(expr $COUNT + 1) + echo "\n>>>>> checking kconfig number $COUNT <<<<<" + coverage run -a --branch bin/kconfig-hardened-check -c $C > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -m verbose > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l /proc/cmdline > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l ./cmdline_example > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l ./cmdline_example -m verbose > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l ./cmdline_example -m json > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l ./cmdline_example -m show_ok > /dev/null + coverage run -a --branch bin/kconfig-hardened-check -c $C -l ./cmdline_example -m show_fail > /dev/null +done +echo "\n>>>>> have checked $COUNT kconfigs <<<<<" + +echo "Collect coverage for error handling" +echo ">>>>> lonely -l <<<<<" +! coverage run -a --branch bin/kconfig-hardened-check -l /proc/cmdline + +echo ">>>>> wrong modes for -p <<<<<" +! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_ok +! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_fail + +echo ">>>>> -p and -c together <<<<<" +! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -c kconfig_hardened_check/config_files/distros/fedora_34.config + +cp kconfig_hardened_check/config_files/distros/fedora_34.config ./test.config + +echo ">>>>> no kernel version <<<<<" +sed '3d' test.config > error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> strange kernel version string <<<<<" +sed '3 s/5./version 5./' test.config > error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> no arch <<<<<" +sed '305d' test.config > error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> more than one arch <<<<<" +cp test.config error.config +echo 'CONFIG_ARM64=y' >> error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> invalid disabled kconfig option <<<<<" +sed '28 s/is not set/is not set yet/' test.config > error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> invalid enabled kconfig option <<<<<" +cp test.config error.config +echo 'CONFIG_FOO=is not set' >> error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> one config option multiple times <<<<<" +cp test.config error.config +echo 'CONFIG_BUG=y' >> error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> invalid compiler versions <<<<<" +cp test.config error.config +sed '8 s/CONFIG_CLANG_VERSION=0/CONFIG_CLANG_VERSION=120000/' test.config > error.config +! coverage run -a --branch bin/kconfig-hardened-check -c error.config + +echo ">>>>> multi-line cmdline file <<<<<" +echo 'hey man 1' > cmdline +echo 'hey man 2' >> cmdline +! coverage run -a --branch bin/kconfig-hardened-check -c test.config -l cmdline + +echo "The end of the functional tests" diff --git a/.woodpecker/functional_test.yml b/.woodpecker/functional_test.yml index ebd631a..a41068e 100644 --- a/.woodpecker/functional_test.yml +++ b/.woodpecker/functional_test.yml @@ -16,7 +16,17 @@ pipeline: - kconfig-hardened-check - echo "Check all configs with the installed tool..." - CONFIG_DIR=`find /usr/local/lib/ -name config_files` - - KCONFIGS=`find $CONFIG_DIR -type f | grep "\.config"` + - KCONFIGS=`find $CONFIG_DIR -type f | grep -e "\.config" -e "\.gz"` - COUNT=0 - for C in $KCONFIGS; do COUNT=$(expr $COUNT + 1); echo ">>>>> checking kconfig number $COUNT <<<<<"; kconfig-hardened-check -c $C -l /proc/cmdline; done - echo ">>>>> have checked $COUNT kconfigs <<<<<" + functional-test-with-coverage: + image: python:3 + pull: true + commands: + - echo "Install the coverage tool..." + - python --version + - pip install --no-cache-dir coverage + - echo "Run the functional tests and collect the coverage..." + - sh .github/workflows/functional_test.sh + - coverage report -- 2.31.1