diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/README | 92 | ||||
| -rw-r--r-- | test/py/conftest.py | 1 | ||||
| -rw-r--r-- | test/py/multiplexed_log.py | 10 | ||||
| -rw-r--r-- | test/py/tests/test_vboot.py | 185 | ||||
| -rw-r--r-- | test/py/tests/vboot/sandbox-kernel.dts (renamed from test/vboot/sandbox-kernel.dts) | 0 | ||||
| -rw-r--r-- | test/py/tests/vboot/sandbox-u-boot.dts (renamed from test/vboot/sandbox-u-boot.dts) | 0 | ||||
| -rw-r--r-- | test/py/tests/vboot/sign-configs-sha1.its (renamed from test/vboot/sign-configs-sha1.its) | 0 | ||||
| -rw-r--r-- | test/py/tests/vboot/sign-configs-sha256.its (renamed from test/vboot/sign-configs-sha256.its) | 0 | ||||
| -rw-r--r-- | test/py/tests/vboot/sign-images-sha1.its (renamed from test/vboot/sign-images-sha1.its) | 0 | ||||
| -rw-r--r-- | test/py/tests/vboot/sign-images-sha256.its (renamed from test/vboot/sign-images-sha256.its) | 0 | ||||
| -rw-r--r-- | test/py/u_boot_console_base.py | 16 | ||||
| -rw-r--r-- | test/py/u_boot_console_sandbox.py | 2 | ||||
| -rw-r--r-- | test/py/u_boot_utils.py | 39 | ||||
| -rwxr-xr-x | test/run | 4 | ||||
| -rw-r--r-- | test/vboot/.gitignore | 3 | ||||
| -rwxr-xr-x | test/vboot/vboot_test.sh | 151 |
16 files changed, 345 insertions, 158 deletions
diff --git a/test/README b/test/README new file mode 100644 index 0000000000..ee559729b0 --- /dev/null +++ b/test/README @@ -0,0 +1,92 @@ +Testing in U-Boot +================= + +U-Boot has a large amount of code. This file describes how this code is +tested and what tests you should write when adding a new feature. + + +Running tests +------------- + +To run most tests on sandbox, type this: + + test/run + +in the U-Boot directory. Note that only the pytest suite is run using this +comment. + + +Sandbox +------- +U-Boot can be built as a user-space application (e.g. for Linux). This +allows test to be executed without needing target hardware. The 'sandbox' +target provides this feature and it is widely used in tests. + + +Pytest Suite +------------ + +Many tests are available using the pytest suite, in test/py. This can run +either on sandbox or on real hardware. It relies on the U-Boot console to +inject test commands and check the result. It is slower to run than C code, +but provides the ability to unify lots of test and summarise their results. + +You can run the tests on sandbox with: + + ./test/py/test.py --bd sandbox --build + +This will produce HTML output in build-sandbox/test-log.html + +See test/py/README.md for more information about the pytest suite. + + +tbot +---- + +Tbot provides a way to execute tests on target hardware. It is intended for +trying out both U-Boot and Linux (and potentially other software) on a +number of boards automatically. It can be used to create a continuous test +environment. See tools/tbot/README for more information. + + +Ad-hoc tests +------------ + +There are several ad-hoc tests which run outside the pytest environment: + + test/fs - File system test (shell script) + test/image - FIT and lagacy image tests (shell script and Python) + test/stdint - A test that stdint.h can be used in U-Boot (shell script) + trace - Test for the tracing feature (shell script) + +The above should be converted to run as part of the pytest suite. + + +When to write tests +------------------- + +If you add code to U-Boot without a test you are taking a risk. Even if you +perform thorough manual testing at the time of submission, it may break when +future changes are made to U-Boot. It may even break when applied to mainline, +if other changes interact with it. A good mindset is that untested code +probably doesn't work and should be deleted. + +You can assume that the Pytest suite will be run before patches are accepted +to mainline, so this provides protection against future breakage. + +On the other hand there is quite a bit of code that is not covered with tests, +or is covered sparingly. So here are some suggestions: + +- If you are adding a new uclass, add a sandbox driver and a test that uses it +- If you are modifying code covered by an existing test, add a new test case + to cover your changes +- If the code you are modifying has not tests, consider writing one. Even a + very basic test is useful, and may be picked up and enhanced by others. It + is much easier to add onto a test - writing a new large test can seem + daunting to most contributors. + + +Future work +----------- + +Converting existing shell scripts into pytest tests. diff --git a/test/py/conftest.py b/test/py/conftest.py index 050f6e409d..5b3a92316b 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -179,6 +179,7 @@ def pytest_configure(config): ubconfig.board_type = board_type ubconfig.board_identity = board_identity ubconfig.gdbserver = gdbserver + ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb' env_vars = ( 'board_type', diff --git a/test/py/multiplexed_log.py b/test/py/multiplexed_log.py index 68917eb0ea..35a32fb5c0 100644 --- a/test/py/multiplexed_log.py +++ b/test/py/multiplexed_log.py @@ -101,6 +101,7 @@ class RunAndLog(object): self.logfile = logfile self.name = name self.chained_file = chained_file + self.output = None def close(self): """Clean up any resources managed by this object.""" @@ -109,6 +110,9 @@ class RunAndLog(object): def run(self, cmd, cwd=None, ignore_errors=False): """Run a command as a sub-process, and log the results. + The output is available at self.output which can be useful if there is + an exception. + Args: cmd: The command to execute. cwd: The directory to run the command in. Can be None to use the @@ -119,7 +123,7 @@ class RunAndLog(object): raised if such problems occur. Returns: - Nothing. + The output as a string. """ msg = '+' + ' '.join(cmd) + '\n' @@ -159,8 +163,12 @@ class RunAndLog(object): self.logfile.write(self, output) if self.chained_file: self.chained_file.write(output) + + # Store the output so it can be accessed if we raise an exception. + self.output = output if exception: raise exception + return output class SectionCtxMgr(object): """A context manager for Python's "with" statement, which allows a certain diff --git a/test/py/tests/test_vboot.py b/test/py/tests/test_vboot.py new file mode 100644 index 0000000000..c77989591c --- /dev/null +++ b/test/py/tests/test_vboot.py @@ -0,0 +1,185 @@ +# Copyright (c) 2013, Google Inc. +# +# SPDX-License-Identifier: GPL-2.0+ +# +# U-Boot Verified Boot Test + +""" +This tests verified boot in the following ways: + +For image verification: +- Create FIT (unsigned) with mkimage +- Check that verification shows that no keys are verified +- Sign image +- Check that verification shows that a key is now verified + +For configuration verification: +- Corrupt signature and check for failure +- Create FIT (with unsigned configuration) with mkimage +- Check that image veriication works +- Sign the FIT and mark the key as 'required' for verification +- Check that image verification works +- Corrupt the signature +- Check that image verification no-longer works + +Tests run with both SHA1 and SHA256 hashing. +""" + +import pytest +import sys +import u_boot_utils as util + +@pytest.mark.buildconfigspec('fit_signature') +def test_vboot(u_boot_console): + """Test verified boot signing with mkimage and verification with 'bootm'. + + This works using sandbox only as it needs to update the device tree used + by U-Boot to hold public keys from the signing process. + + The SHA1 and SHA256 tests are combined into a single test since the + key-generation process is quite slow and we want to avoid doing it twice. + """ + def dtc(dts): + """Run the device-tree compiler to compile a .dts file + + The output file will be the same as the input file but with a .dtb + extension. + + Args: + dts: Device tree file to compile. + """ + dtb = dts.replace('.dts', '.dtb') + util.cmd(cons, 'dtc %s %s%s -O dtb ' + '-o %s%s' % (dtc_args, datadir, dts, tmpdir, dtb)) + + def run_bootm(test_type, expect_string): + """Run a 'bootm' command U-Boot. + + This always starts a fresh U-Boot instance since the device tree may + contain a new public key. + + Args: + test_type: A string identifying the test type + expect_string: A string which is expected in the output + """ + cons.cleanup_spawn() + cons.ensure_spawned() + cons.log.action('%s: Test Verified Boot Run: %s' % (algo, test_type)) + output = cons.run_command_list( + ['sb load hostfs - 100 %stest.fit' % tmpdir, + 'fdt addr 100', + 'bootm 100']) + assert(expect_string in output) + + def make_fit(its): + """Make a new FIT from the .its source file + + This runs 'mkimage -f' to create a new FIT. + + Args: + its: Filename containing .its source + """ + util.run_and_log(cons, [mkimage, '-D', dtc_args, '-f', + '%s%s' % (datadir, its), fit]) + + def sign_fit(): + """Sign the FIT + + Signs the FIT and writes the signature into it. It also writes the + public key into the dtb. + """ + cons.log.action('%s: Sign images' % algo) + util.run_and_log(cons, [mkimage, '-F', '-k', tmpdir, '-K', dtb, + '-r', fit]) + + def test_with_algo(sha): + """Test verified boot with the given hash algorithm + + This is the main part of the test code. The same procedure is followed + for both hashing algorithms. + + Args: + sha: Either 'sha1' or 'sha256', to select the algorithm to use + """ + global algo + + algo = sha + + # Compile our device tree files for kernel and U-Boot + dtc('sandbox-kernel.dts') + dtc('sandbox-u-boot.dts') + + # Build the FIT, but don't sign anything yet + cons.log.action('%s: Test FIT with signed images' % algo) + make_fit('sign-images-%s.its' % algo) + run_bootm('unsigned images', 'dev-') + + # Sign images with our dev keys + sign_fit() + run_bootm('signed images', 'dev+') + + # Create a fresh .dtb without the public keys + dtc('sandbox-u-boot.dts') + + cons.log.action('%s: Test FIT with signed configuration' % algo) + make_fit('sign-configs-%s.its' % algo) + run_bootm('unsigned config', '%s+ OK' % algo) + + # Sign images with our dev keys + sign_fit() + run_bootm('signed config', 'dev+') + + cons.log.action('%s: Check signed config on the host' % algo) + + util.run_and_log(cons, [fit_check_sign, '-f', fit, '-k', tmpdir, + '-k', dtb]) + + # Increment the first byte of the signature, which should cause failure + sig = util.cmd(cons, 'fdtget -t bx %s %s value' % (fit, sig_node)) + byte_list = sig.split() + byte = int(byte_list[0], 16) + byte_list = ['%x' % (byte + 1)] + byte_list[1:] + sig = ' '.join(byte_list) + util.cmd(cons, 'fdtput -t bx %s %s value %s' % (fit, sig_node, sig)) + + run_bootm('Signed config with bad hash', 'Bad Data Hash') + + cons.log.action('%s: Check bad config on the host' % algo) + util.run_and_log_expect_exception(cons, [fit_check_sign, '-f', fit, + '-k', dtb], 1, 'Failed to verify required signature') + + cons = u_boot_console + tmpdir = cons.config.result_dir + '/' + tmp = tmpdir + 'vboot.tmp' + datadir = 'test/py/tests/vboot/' + fit = '%stest.fit' % tmpdir + mkimage = cons.config.build_dir + '/tools/mkimage' + fit_check_sign = cons.config.build_dir + '/tools/fit_check_sign' + dtc_args = '-I dts -O dtb -i %s' % tmpdir + dtb = '%ssandbox-u-boot.dtb' % tmpdir + sig_node = '/configurations/conf@1/signature@1' + + # Create an RSA key pair + public_exponent = 65537 + util.cmd(cons, 'openssl genpkey -algorithm RSA -out %sdev.key ' + '-pkeyopt rsa_keygen_bits:2048 ' + '-pkeyopt rsa_keygen_pubexp:%d ' + '2>/dev/null' % (tmpdir, public_exponent)) + + # Create a certificate containing the public key + util.cmd(cons, 'openssl req -batch -new -x509 -key %sdev.key -out ' + '%sdev.crt' % (tmpdir, tmpdir)) + + # Create a number kernel image with zeroes + with open('%stest-kernel.bin' % tmpdir, 'w') as fd: + fd.write(5000 * chr(0)) + + try: + # We need to use our own device tree file. Remember to restore it + # afterwards. + old_dtb = cons.config.dtb + cons.config.dtb = dtb + test_with_algo('sha1') + test_with_algo('sha256') + finally: + cons.config.dtb = old_dtb diff --git a/test/vboot/sandbox-kernel.dts b/test/py/tests/vboot/sandbox-kernel.dts index a1e853c9ca..a1e853c9ca 100644 --- a/test/vboot/sandbox-kernel.dts +++ b/test/py/tests/vboot/sandbox-kernel.dts diff --git a/test/vboot/sandbox-u-boot.dts b/test/py/tests/vboot/sandbox-u-boot.dts index 63f8f401de..63f8f401de 100644 --- a/test/vboot/sandbox-u-boot.dts +++ b/test/py/tests/vboot/sandbox-u-boot.dts diff --git a/test/vboot/sign-configs-sha1.its b/test/py/tests/vboot/sign-configs-sha1.its index db2ed79355..db2ed79355 100644 --- a/test/vboot/sign-configs-sha1.its +++ b/test/py/tests/vboot/sign-configs-sha1.its diff --git a/test/vboot/sign-configs-sha256.its b/test/py/tests/vboot/sign-configs-sha256.its index 1b3432ec14..1b3432ec14 100644 --- a/test/vboot/sign-configs-sha256.its +++ b/test/py/tests/vboot/sign-configs-sha256.its diff --git a/test/vboot/sign-images-sha1.its b/test/py/tests/vboot/sign-images-sha1.its index f69326a39b..f69326a39b 100644 --- a/test/vboot/sign-images-sha1.its +++ b/test/py/tests/vboot/sign-images-sha1.its diff --git a/test/vboot/sign-images-sha256.its b/test/py/tests/vboot/sign-images-sha256.its index e6aa9fc409..e6aa9fc409 100644 --- a/test/vboot/sign-images-sha256.its +++ b/test/py/tests/vboot/sign-images-sha256.its diff --git a/test/py/u_boot_console_base.py b/test/py/u_boot_console_base.py index 3d2ac44875..4606ad48bf 100644 --- a/test/py/u_boot_console_base.py +++ b/test/py/u_boot_console_base.py @@ -216,6 +216,22 @@ class ConsoleBase(object): self.cleanup_spawn() raise + def run_command_list(self, cmds): + """Run a list of commands. + + This is a helper function to call run_command() with default arguments + for each command in a list. + + Args: + cmd: List of commands (each a string) + Returns: + Combined output of all commands, as a string + """ + output = '' + for cmd in cmds: + output += self.run_command(cmd) + return output + def ctrlc(self): """Send a CTRL-C character to U-Boot. diff --git a/test/py/u_boot_console_sandbox.py b/test/py/u_boot_console_sandbox.py index 6bf4653083..647e1f879f 100644 --- a/test/py/u_boot_console_sandbox.py +++ b/test/py/u_boot_console_sandbox.py @@ -50,7 +50,7 @@ class ConsoleSandbox(ConsoleBase): self.config.build_dir + fname, '-v', '-d', - self.config.build_dir + '/arch/sandbox/dts/test.dtb' + self.config.dtb ] return Spawn(cmd, cwd=self.config.source_dir) diff --git a/test/py/u_boot_utils.py b/test/py/u_boot_utils.py index e4765e38c1..e358c585bf 100644 --- a/test/py/u_boot_utils.py +++ b/test/py/u_boot_utils.py @@ -165,12 +165,47 @@ def run_and_log(u_boot_console, cmd, ignore_errors=False): problems occur. Returns: - Nothing. + The output as a string. """ runner = u_boot_console.log.get_runner(cmd[0], sys.stdout) - runner.run(cmd, ignore_errors=ignore_errors) + output = runner.run(cmd, ignore_errors=ignore_errors) runner.close() + return output + +def cmd(u_boot_console, cmd_str): + """Run a single command string and log its output. + + Args: + u_boot_console: A console connection to U-Boot. + cmd: The command to run, as a string. + + Returns: + The output as a string. + """ + return run_and_log(u_boot_console, cmd_str.split()) + +def run_and_log_expect_exception(u_boot_console, cmd, retcode, msg): + """Run a command which is expected to fail. + + This runs a command and checks that it fails with the expected return code + and exception method. If not, an exception is raised. + + Args: + u_boot_console: A console connection to U-Boot. + cmd: The command to run, as an array of argv[]. + retcode: Expected non-zero return code from the command. + msg: String which should be contained within the command's output. + """ + try: + runner = u_boot_console.log.get_runner(cmd[0], sys.stdout) + runner.run(cmd) + except Exception as e: + assert(msg in runner.output) + else: + raise Exception('Expected exception, but not raised') + finally: + runner.close() ram_base = None def find_ram_base(u_boot_console): diff --git a/test/run b/test/run new file mode 100755 index 0000000000..a6dcf8f44f --- /dev/null +++ b/test/run @@ -0,0 +1,4 @@ +#!/bin/sh + +# Run all tests +./test/py/test.py --bd sandbox --build diff --git a/test/vboot/.gitignore b/test/vboot/.gitignore deleted file mode 100644 index 4631242709..0000000000 --- a/test/vboot/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/*.dtb -/test.fit -/dev-keys diff --git a/test/vboot/vboot_test.sh b/test/vboot/vboot_test.sh deleted file mode 100755 index 6d7abb82bd..0000000000 --- a/test/vboot/vboot_test.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2013, Google Inc. -# -# Simple Verified Boot Test Script -# -# SPDX-License-Identifier: GPL-2.0+ - -set -e - -# Run U-Boot and report the result -# Args: -# $1: Test message -run_uboot() { - echo -n "Test Verified Boot Run: $1: " - ${uboot} -d sandbox-u-boot.dtb >${tmp} -c ' -sb load hostfs - 100 test.fit; -fdt addr 100; -bootm 100; -reset' - if ! grep -q "$2" ${tmp}; then - echo - echo "Verified boot key check failed, output follows:" - cat ${tmp} - false - else - echo "OK" - fi -} - -echo "Simple Verified Boot Test" -echo "=========================" -echo -echo "Please see doc/uImage.FIT/verified-boot.txt for more information" -echo - -err=0 -tmp=/tmp/vboot_test.$$ - -dir=$(dirname $0) - -if [ -z ${O} ]; then - O=. -fi -O=$(readlink -f ${O}) - -dtc="-I dts -O dtb -p 2000" -uboot="${O}/u-boot" -mkimage="${O}/tools/mkimage" -fit_check_sign="${O}/tools/fit_check_sign" -keys="${dir}/dev-keys" -echo ${mkimage} -D "${dtc}" - -echo "Build keys" -mkdir -p ${keys} - -PUBLIC_EXPONENT=${1} - -if [ -z "${PUBLIC_EXPONENT}" ]; then - PUBLIC_EXPONENT=65537 -fi - -# Create an RSA key pair -openssl genpkey -algorithm RSA -out ${keys}/dev.key \ - -pkeyopt rsa_keygen_bits:2048 \ - -pkeyopt rsa_keygen_pubexp:${PUBLIC_EXPONENT} 2>/dev/null - -# Create a certificate containing the public key -openssl req -batch -new -x509 -key ${keys}/dev.key -out ${keys}/dev.crt - -pushd ${dir} >/dev/null - -function do_test { - echo do $sha test - # Compile our device tree files for kernel and U-Boot - dtc -p 0x1000 sandbox-kernel.dts -O dtb -o sandbox-kernel.dtb - dtc -p 0x1000 sandbox-u-boot.dts -O dtb -o sandbox-u-boot.dtb - - # Create a number kernel image with zeroes - head -c 5000 /dev/zero >test-kernel.bin - - # Build the FIT, but don't sign anything yet - echo Build FIT with signed images - ${mkimage} -D "${dtc}" -f sign-images-$sha.its test.fit >${tmp} - - run_uboot "unsigned signatures:" "dev-" - - # Sign images with our dev keys - echo Sign images - ${mkimage} -D "${dtc}" -F -k dev-keys -K sandbox-u-boot.dtb \ - -r test.fit >${tmp} - - run_uboot "signed images" "dev+" - - - # Create a fresh .dtb without the public keys - dtc -p 0x1000 sandbox-u-boot.dts -O dtb -o sandbox-u-boot.dtb - - echo Build FIT with signed configuration - ${mkimage} -D "${dtc}" -f sign-configs-$sha.its test.fit >${tmp} - - run_uboot "unsigned config" $sha"+ OK" - - # Sign images with our dev keys - echo Sign images - ${mkimage} -D "${dtc}" -F -k dev-keys -K sandbox-u-boot.dtb \ - -r test.fit >${tmp} - - run_uboot "signed config" "dev+" - - echo check signed config on the host - if ! ${fit_check_sign} -f test.fit -k sandbox-u-boot.dtb >${tmp}; then - echo - echo "Verified boot key check on host failed, output follows:" - cat ${tmp} - false - else - if ! grep -q "dev+" ${tmp}; then - echo - echo "Verified boot key check failed, output follows:" - cat ${tmp} - false - else - echo "OK" - fi - fi - - run_uboot "signed config" "dev+" - - # Increment the first byte of the signature, which should cause failure - sig=$(fdtget -t bx test.fit /configurations/conf@1/signature@1 value) - newbyte=$(printf %x $((0x${sig:0:2} + 1))) - sig="${newbyte} ${sig:2}" - fdtput -t bx test.fit /configurations/conf@1/signature@1 value ${sig} - - run_uboot "signed config with bad hash" "Bad Data Hash" -} - -sha=sha1 -do_test -sha=sha256 -do_test - -popd >/dev/null - -echo -if ${ok}; then - echo "Test passed" -else - echo "Test failed" -fi |