ci: add .shippable.yml

Add a configuration file for the Shippable continuous integration tool
[1]. This file performs the same steps as the current Travis file
(.travis.yml), but it is faster and simpler. Another advantage is, the
timeout is 1 hour compared to 50 minutes for Travis. All in all, this
could be a good fix for the issues we have with Travis being too slow
to properly check our pull requests.

This was tested on a private fork of optee_os, and it worked well for
verifying pushes to private branches as well as pull requests. A full
build takes about 20-25 minutes, that is including the builds for all
targets plus the xtest run in QEMU (with a fully populated cache).

One reason for Shippable being faster is that a custom Docker container
image is used, namely: jforissier/optee_os_ci on Docker Hub [2].
It is Ununtu 17.04 plus the packages required to build OP-TEE and run
the QEMU regression tests. Therefore, there is no lengthy preparation
step, such as building tools that are missing in the images provided by
Travis. Docker images are cached by Shippable, so our own rarely needs
to be fetched (which takes roughly 5 minutes).

Another reason for the good speed is that we use ccache for everything.
With a warm cache, each platform is built in no more than 5-6 seconds,
and this speedup is not offset by the longer time it takes to persist
a bigger cache file (contrary to what I observed with Travis).

Since caching works so well I also decided to cache the whole QEMU
environment (repo-based tree), so that the repo init + repo sync steps
are usually quite fast (45 seconds).

Lastly, switching GitHub from Travis to Shippable is very easy, so we
may consider doing so in the short term.

Links: [1] https://shippable.com
Links: [2] https://hub.docker.com/r/jforissier/optee_os_ci
Signed-off-by: Jerome Forissier <jerome.forissier@linaro.org>
Reviewed-by: Joakim Bech <joakim.bech@linaro.org>
Acked-by: Etienne Carriere <etienne.carriere@linaro.org>

1 files changed, 186 insertions, 0 deletions

diff --git a/.shippable.yml b/.shippable.yml
new file mode 100644
index 00000000..15c821a8
--- /dev/null
+++ b/.shippable.yml
@@ -0,0 +1,186 @@
+language: c
+build:
+  cache: true
+  cache_dir_list:
+    - ${SHIPPABLE_BUILD_DIR}/.ccache
+    - ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu
+  pre_ci_boot:
+    image_name: jforissier/optee_os_ci
+    image_tag: latest
+    pull: true
+    options: "-e HOME=/root"
+  ci:
+    - export LC_ALL=C
+    - export CROSS_COMPILE32="ccache arm-linux-gnueabihf-"
+    - export CROSS_COMPILE64="ccache aarch64-linux-gnu-"
+    - export CCACHE_DIR=${SHIPPABLE_BUILD_DIR}/.ccache
+    - function _make() { make -j$(getconf _NPROCESSORS_ONLN) -s $*; ccache -s; ccache -z; }
+    - ccache -z
+
+    #
+    # Checkpatch
+    #
+
+    # Download the Linux checkpatch tool
+    - >
+      mkdir ${SHIPPABLE_BUILD_DIR}/checkpatch &&
+      cd ${SHIPPABLE_BUILD_DIR}/checkpatch &&
+      wget https://raw.githubusercontent.com/torvalds/linux/master/scripts/checkpatch.pl &&
+      chmod a+x checkpatch.pl &&
+      wget https://raw.githubusercontent.com/torvalds/linux/master/scripts/spelling.txt &&
+      echo "invalid.struct.name" >const_structs.checkpatch &&
+      export PATH=${SHIPPABLE_BUILD_DIR}/checkpatch:${PATH} &&
+      source ${SHIPPABLE_BUILD_DIR}/scripts/checkpatch_inc.sh;
+    # Run checkpatch on:
+    # - the tip of the branch if we're not in a pull request
+    # - each commit in the development branch that's not in the target branch otherwise
+    # Then, if we have a pull request with more than 1 commit, also check the squashed commits
+    # Useful to check if fix-up commits do indeed solve previous checkpatch errors.
+    # Note: error reporting is deferred ($checkpatch_failed=1) to avoid stopping the build.
+    - echo Checkpatch ;
+      cd ${SHIPPABLE_BUILD_DIR} ;
+      if [ "$IS_PULL_REQUEST" == "false" ]; then
+        echo "Running checkpatch on branch HEAD:";
+        checkpatch HEAD || export checkpatch_failed=1;
+      else
+        echo "Running checkpatch on each patch in pull request:";
+        for c in $(git rev-list --reverse ${SHIPPABLE_COMMIT_RANGE}); do
+          checkpatch $c || export checkpatch_failed=1;
+        done;
+      fi;
+      if [ "$IS_PULL_REQUEST" == "true" ]; then
+        if [ "$(git rev-list --count ${SHIPPABLE_COMMIT_RANGE})" -gt 1 ]; then
+          echo "Running checkpatch on gobal diff (squashed commits):";
+          checkdiff $(git rev-list ${SHIPPABLE_COMMIT_RANGE} | tail -1) $(git rev-list ${SHIPPABLE_COMMIT_RANGE} | head -1) || export checkpatch_failed=1;
+        fi;
+      fi;
+
+    #
+    # Build tests
+    #
+
+    - _make
+    - _make CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make CFG_TEE_CORE_LOG_LEVEL=3 DEBUG=1
+    - _make CFG_TEE_CORE_LOG_LEVEL=2 DEBUG=1
+    - _make CFG_TEE_CORE_LOG_LEVEL=1 CFG_TEE_CORE_DEBUG=y DEBUG=1
+    - _make CFG_TEE_CORE_LOG_LEVEL=1 CFG_TEE_CORE_DEBUG=n DEBUG=0
+    - _make CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=y DEBUG=1
+    - _make CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n DEBUG=0
+    - _make CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make CFG_TEE_CORE_MALLOC_DEBUG=y
+    - _make CFG_CORE_SANITIZE_UNDEFINED=y
+    - _make CFG_CORE_SANITIZE_KADDRESS=y
+    - _make CFG_CRYPTO=n
+    - _make CFG_CRYPTO_{AES,DES}=n
+    - _make CFG_CRYPTO_{DSA,RSA,DH}=n
+    - _make CFG_CRYPTO_{DSA,RSA,DH,ECC}=n
+    - _make CFG_CRYPTO_{H,C,CBC_}MAC=n
+    - _make CFG_CRYPTO_{G,C}CM=n
+    - _make CFG_CRYPTO_{MD5,SHA{1,224,256,384,512}}=n
+    - _make CFG_CRYPTO=n CFG_CRYPTO_ECC=y
+    - _make CFG_WITH_PAGER=y
+    - _make CFG_WITH_PAGER=y CFG_TEE_CORE_DEBUG=y
+    - _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y
+    - _make CFG_WITH_LPAE=y
+    - _make CFG_WITH_STATS=y
+    - _make CFG_RPMB_FS=y
+    - _make CFG_RPMB_FS=y CFG_RPMB_TESTKEY=y
+    - _make CFG_REE_FS=n CFG_RPMB_FS=y
+    - _make CFG_WITH_USER_TA=n CFG_CRYPTO=n CFG_SE_API=n CFG_PCSC_PASSTHRU_READER_DRV=n
+    - _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y CFG_RPMB_FS=y CFG_DT=y CFG_PS2MOUSE=y CFG_PL050=y CFG_PL111=y CFG_TEE_CORE_LOG_LEVEL=1 CFG_TEE_CORE_DEBUG=y DEBUG=1
+    - _make CFG_WITH_PAGER=y CFG_WITH_LPAE=y CFG_RPMB_FS=y CFG_DT=y CFG_PS2MOUSE=y CFG_PL050=y CFG_PL111=y CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_CORE_DEBUG=n DEBUG=0
+    - _make CFG_BUILT_IN_ARGS=y CFG_PAGEABLE_ADDR=0 CFG_NS_ENTRY_ADDR=0 CFG_DT_ADDR=0 CFG_DT=y
+    - _make CFG_TA_GPROF_SUPPORT=y CFG_ULIBS_GPROF=y
+    - _make CFG_SECURE_DATA_PATH=y
+    - _make PLATFORM=vexpress-qemu_armv8a CFG_ARM64_core=y
+    - _make PLATFORM=vexpress-qemu_armv8a CFG_ARM64_core=y CFG_WITH_PAGER=y
+    - _make PLATFORM=vexpress-qemu_armv8a CFG_ARM64_core=y CFG_RPMB_FS=y
+    - _make PLATFORM=vexpress-qemu_armv8a CFG_ARM64_core=y CFG_TA_GPROF_SUPPORT=y CFG_ULIBS_GPROF=y
+    - _make PLATFORM=stm-b2260
+    - _make PLATFORM=stm-b2260 CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=stm-b2260 CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make PLATFORM=stm-cannes
+    - _make PLATFORM=stm-cannes CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=stm-cannes CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make PLATFORM=vexpress-fvp CFG_ARM32_core=y
+    - _make PLATFORM=vexpress-fvp CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1 CFG_TZC400=y
+    - _make PLATFORM=vexpress-fvp CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0 CFG_TZC400=y
+    - _make PLATFORM=vexpress-fvp CFG_ARM64_core=y
+    - _make PLATFORM=vexpress-fvp CFG_ARM64_core=y CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1 CFG_TZC400=y
+    - _make PLATFORM=vexpress-fvp CFG_ARM64_core=y CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0 CFG_TZC400=y
+    - _make PLATFORM=vexpress-juno
+    - _make PLATFORM=vexpress-juno CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=vexpress-juno CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make PLATFORM=vexpress-juno CFG_ARM64_core=y
+    - _make PLATFORM=vexpress-juno CFG_ARM64_core=y CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=vexpress-juno CFG_ARM64_core=y CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make PLATFORM=sunxi CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=sunxi CFG_TEE_CORE_LOG_LEVEL=0 CFG_TEE_TA_LOG_LEVEL=0 DEBUG=0
+    - _make PLATFORM=hikey
+    - _make PLATFORM=hikey CFG_ARM64_core=y
+    - _make PLATFORM=hikey CFG_ARM64_core=y CFG_TEE_TA_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=mediatek-mt8173 CFG_ARM64_core=y
+    - _make PLATFORM=imx-mx6ulevk ARCH=arm CFG_PAGEABLE_ADDR=0 CFG_NS_ENTRY_ADDR=0x80800000 CFG_DT_ADDR=0x83000000 CFG_DT=y DEBUG=y CFG_TEE_CORE_LOG_LEVEL=4
+    - _make PLATFORM=imx-mx6ullevk ARCH=arm CFG_PAGEABLE_ADDR=0 CFG_NS_ENTRY_ADDR=0x80800000 CFG_DT=y DEBUG=y CFG_TEE_CORE_LOG_LEVEL=4
+    - _make PLATFORM=imx-mx6qsabrelite
+    - _make PLATFORM=imx-mx6qsabresd
+    - _make PLATFORM=imx-mx6dlsabresd
+    - _make PLATFORM=imx-mx7dsabresd
+    - _make PLATFORM=ti-dra7xx
+    - _make PLATFORM=ti-am57xx
+    - _make PLATFORM=ti-am43xx
+    - _make PLATFORM=sprd-sc9860
+    - _make PLATFORM=sprd-sc9860 CFG_ARM64_core=y
+    - _make PLATFORM=ls-ls1021atwr
+    - _make PLATFORM=ls-ls1021aqds
+    - _make PLATFORM=ls-ls1043ardb CFG_ARM64_core=y
+    - _make PLATFORM=ls-ls1046ardb CFG_ARM64_core=y
+    - _make PLATFORM=zynq7k-zc702
+    - _make PLATFORM=zynqmp-zcu102
+    - _make PLATFORM=zynqmp-zcu102 CFG_ARM64_core=y
+    - _make PLATFORM=d02
+    - _make PLATFORM=d02 CFG_ARM64_core=y
+    - _make PLATFORM=rcar
+    - _make PLATFORM=rcar CFG_ARM64_core=y
+    - _make PLATFORM=rpi3
+    - _make PLATFORM=rpi3 CFG_ARM64_core=y
+    - _make PLATFORM=hikey-hikey960
+    - _make PLATFORM=hikey-hikey960 CFG_ARM64_core=y
+    - _make PLATFORM=hikey-hikey960 CFG_ARM64_core=y CFG_TEE_TA_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=rockchip-rk322x
+    - _make PLATFORM=rockchip-rk322x CFG_TEE_CORE_LOG_LEVEL=4 DEBUG=1
+    - _make PLATFORM=sam
+
+    #
+    # Regressions tests (QEMU)
+    #
+
+    # Download Google's repo script
+    - >
+      mkdir -p ${SHIPPABLE_BUILD_DIR}/bin &&
+      export PATH=${SHIPPABLE_BUILD_DIR}/bin:${PATH} &&
+      curl https://storage.googleapis.com/git-repo-downloads/repo >${SHIPPABLE_BUILD_DIR}/bin/repo &&
+      chmod +x ${SHIPPABLE_BUILD_DIR}/bin/repo;
+    # Use repo to clone the OP-TEE CI environment for QEMU and link optee_os to the current checkout
+    - >
+      mkdir -p ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu &&
+      cd ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu &&
+      repo init -u https://github.com/OP-TEE/manifest.git -m travis.xml </dev/null &&
+      repo sync -d --force-sync --no-clone-bundle --no-tags --quiet -j 2 &&
+      cd ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu/qemu &&
+      git submodule update --init dtc &&
+      rm -rf ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu/optee_os &&
+      ln -s ${SHIPPABLE_BUILD_DIR} ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu/optee_os;
+    # Run xtest in QEMU
+    - >
+      cd ${SHIPPABLE_BUILD_DIR}/optee_repo_qemu/build &&
+      unset CC &&
+      make -k clean 2>&1 >/dev/null &&
+      ccache -z &&
+      make -s -j$(getconf _NPROCESSORS_ONLN) check CROSS_COMPILE="ccache arm-linux-gnueabihf-" AARCH32_CROSS_COMPILE=arm-linux-gnueabihf- DUMP_LOGS_ON_ERROR=1 &&
+      ccache -s &&
+      make -k clean 2>&1 >/dev/null;
+
+    # Make the ci step fail if we had a checkpatch error
+    - test -z "${checkpatch_failed}" || (echo "There were checkpatch error(s), please check the 'Checkpatch' section above"; false)