Update and cleanup docker files

resources/docker_files/README.md

@@ -36,6 +36,8 @@ sudo docker run --network=host --rm -i -t ubuntu-18.04
 ```
 Above, ```-i``` is for interactive mode and ```-t``` is for emulating a tty. ```--rm``` tells docker to cleanup the container after exit. (See note above regarding `--network=host`.) All images launch ```bash``` on startup. Hence, user is on a ```bash``` shell when image is started in the interactive mode.
 
+Note that the additional parameter `--security-opt seccomp=unconfined` which is disabling the ASLR for the zeroize test, is now set by default.
+
 Use ```run.sh``` for enabling ```git``` and mounting a host workspace inside docker. Example:
 ```sh
 $ ./run.sh /home/mazimkhan/github/mazimkhan ubuntu-18.04

resources/docker_files/run.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-#  Copyright (c) 2017-2021, ARM Limited, All Rights Reserved
+#  Copyright (c) 2017-2022, ARM Limited, All Rights Reserved
 #  SPDX-License-Identifier: Apache-2.0
 #
 #  Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -56,5 +56,5 @@ echo "  Mounting $SSH_CFG_PATH --> /home/user/.ssh"
 echo "  Mounting $MOUNT_DIR --> /var/lib/ws"
 echo "****************************************************"
 
-sudo docker run --network=host --rm -i -t -u $USR_ID:$USR_GRP -w /var/lib/ws -v $MOUNT_DIR:/var/lib/ws -v $SSH_CFG_PATH:/home/user/.ssh --cap-add SYS_PTRACE ${IMAGE}
+sudo docker run --network=host --rm -i -t -u $USR_ID:$USR_GRP -w /var/lib/ws -v $MOUNT_DIR:/var/lib/ws -v $SSH_CFG_PATH:/home/user/.ssh --cap-add SYS_PTRACE --security-opt seccomp=unconfined ${IMAGE}
 

resources/docker_files/ubuntu-16.04/Dockerfile

@@ -1,6 +1,6 @@
 # ubuntu-16.04/Dockerfile
 #
-#  Copyright (c) 2018-2021, ARM Limited, All Rights Reserved
+#  Copyright (c) 2018-2022, ARM Limited, All Rights Reserved
 #  SPDX-License-Identifier: Apache-2.0
 #
 #  Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -117,7 +117,7 @@ RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA-
 
 ENV ARMC5_BIN_DIR=/usr/local/ARM_Compiler_5.06u3/bin/
 ENV PATH=$PATH:/usr/local/ARM_Compiler_5.06u3/bin
-ENV ARMLMD_LICENSE_FILE=[email protected]
+ENV ARMLMD_LICENSE_FILE=[email protected]:[email protected]:[email protected]:[email protected]
 
 # Install ARM Compiler 6.6
 RUN mkdir temp && cd temp && \

resources/docker_files/ubuntu-20.04/Dockerfile

@@ -0,0 +1,269 @@
+# ubuntu-20.04/Dockerfile
+#
+#  Copyright (c) 2018-2022, ARM Limited, All Rights Reserved
+#  SPDX-License-Identifier: Apache-2.0
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  This file is part of Mbed TLS (https://www.trustedfirmware.org/projects/mbed-tls/)
+
+# Purpose
+# -------
+#
+# This docker file is for creating the ubuntu-20.04 image that is used in the
+# CI. It can also be used for reproducing and testing CI failures.
+
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+WORKDIR /opt/src
+
+# Support for i386:
+# - for 32-bit builds+tests of Mbed TLS
+# - required to install Arm Compiler 5.06 (armcc)
+RUN dpkg --add-architecture i386
+
+# Main apt-get call with all packages except those that have conflicts,
+# handled below. One big alphabetised list, in order to avoid duplicates, with
+# comments explaining why each package is needed.
+RUN apt-get update -q && apt-get install -yq \
+        # installed from source, but this installs the dependencies
+        abi-dumper \
+        # to build Mbed TLS: gcc, binutils, make, etc.
+        build-essential \
+        # to build Mbed TLS
+        clang \
+        # to build Mbed TLS
+        cmake \
+        # to build Mbed TLS's documentation
+        doxygen \
+        # to cross-build Mbed TLS
+        gcc-mingw-w64-i686 \
+        # to check out Mbed TLS and others
+        git \
+        # to build Mbed TLS's documentation
+        graphviz \
+        # to measure code coverage of Mbed TLS
+        lcov \
+        # for 32-bit Mbed TLS testing and armcc
+        libc6-i386 \
+        # for 32-bit Mbed TLS testing and armcc
+        libc6:i386 \
+        # to build GnuTLS (nettle with public key support aka hogweed)
+        libgmp-dev \
+        # to build GnuTLS >= 3.6 (could also use --with-included-unistring)
+        libunistring-dev \
+        # for armcc
+        libstdc++6:i386 \
+        # to build GnuTLS
+        libtasn1-6-dev \
+        # needed for armcc (see locale-gen below)
+        locales \
+        # used by compat.sh and ssl-opt.sh
+        lsof \
+        # to build GnuTLS (nettle)
+        m4 \
+        # to build Mbed TLS and others
+        make \
+        # to build GnuTLS with locally-compiled nettle
+        pkg-config \
+        # to install the preferred version of pylint
+        python3-pip \
+        # for Mbed TLS tests
+        valgrind \
+        # to download things installed from other places
+        wget \
+        # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0)
+        zlib1g \
+        # to build Mbed TLS with MBEDTLS_ZILIB_SUPPORT (removed in 3.0)
+        zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/
+
+# Install all the parts of gcc-multilib, which is necessary for 32-bit builds.
+# gcc-multilib conflicts with cross-compiler packages that we'll install later,
+# so don't keep it around. Just let it install its dependencies
+# (gcc-<VERSION>-multilib and libc support), then remove it. Manually create
+# one crucial symlink that's otherwise provided by the gcc-multilib package
+# (without that symlink, 32-bit builds won't find system headers). Note that
+# just installing the dependencies of gcc-multilib also brings in gcc-multilib
+# as a Recommends dependency.
+RUN apt-get update -q && apt-get install -yq \
+        gcc-multilib \
+    && rm -rf /var/lib/apt/lists/ && \
+    dpkg -r gcc-multilib && \
+    ln -s x86_64-linux-gnu/asm /usr/include/asm
+
+# Install arm-linux-gnueabi-gcc - to cross-build Mbed TLS
+RUN apt-get update -q && apt-get install -yq \
+        gcc-arm-linux-gnueabi \
+        libc6-dev-armel-cross \
+    && rm -rf /var/lib/apt/lists/
+
+# Install ARM Compiler 5.06
+RUN wget -q https://developer.arm.com/-/media/Files/downloads/compiler/DS500-PA-00003-r5p0-22rel0.tgz && \
+    tar -zxf DS500-PA-00003-r5p0-22rel0.tgz && \
+    ./Installer/setup.sh --i-agree-to-the-contained-eula --no-interactive -d /usr/local/ARM_Compiler_5.06u3 --quiet && \
+    rm -rf DS500-PA-00003-r5p0-22rel0.tgz releasenotes.html Installer/
+
+ENV ARMC5_BIN_DIR=/usr/local/ARM_Compiler_5.06u3/bin/
+ENV PATH=$PATH:/usr/local/ARM_Compiler_5.06u3/bin
+ENV [email protected]:[email protected]:[email protected]:[email protected]
+
+# Install ARM Compiler 6.6
+RUN mkdir temp && cd temp && \
+    wget -q --no-check-certificate https://developer.arm.com/-/media/Files/downloads/compiler/DS500-BN-00026-r5p0-07rel0.tgz?revision=8f0d9fb0-9616-458c-b2f5-d0dac83ea93c?product=Downloads,64-bit,,Linux,6.6 -O arm6.tgz && \
+    tar -zxf arm6.tgz  && ls -ltr && \
+    ./install_x86_64.sh --i-agree-to-the-contained-eula --no-interactive -d /usr/local/ARM_Compiler_6.6 --quiet && \
+    cd .. && rm -rf temp/
+
+ENV ARMC6_BIN_DIR=/usr/local/ARM_Compiler_6.6/bin/
+
+# Install arm-none-eabi-gcc
+RUN wget -q https://developer.arm.com/-/media/Files/downloads/gnu-rm/5_4-2016q3/gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -O gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 && \
+    tar -xjf gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2 -C /opt && \
+    rm gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2
+
+ENV PATH=/opt/gcc-arm-none-eabi-5_4-2016q3/bin:$PATH
+
+# Install abi-compliance-checker
+RUN apt-get update -q && apt-get install -yq abi-compliance-checker
+
+# Install abi-dumper
+RUN apt-get install -yq abi-dumper
+
+# Install exact upstream versions of OpenSSL and GnuTLS
+#
+# Distro packages tend to include patches that disrupt our testing scripts,
+# and such patches may be added at any time. Avoid surprises by using fixed
+# versions.
+#
+# GnuTLS has a number of (optional) dependencies:
+# - nettle (crypto library): quite tighly coupled, so build one for each
+# version of GnuTLS that we want.
+# - libtasn1: can use the Ubuntu version
+# - p11-kit: optional, for smart-card support - configure it out
+# - libunistring: since 3.6 - the Ubuntu package works; if it didn't a config
+# option --with-included-libunistring is available.
+
+# Install openssl 1.0.2g - main version, in the PATH
+RUN wget -q https://www.openssl.org/source/old/1.0.2/openssl-1.0.2g.tar.gz && \
+    tar -zxf openssl-1.0.2g.tar.gz && cd openssl-1.0.2g && \
+    ./config --openssldir=/usr/local/openssl-1.0.2g enable-ssl-trace && \
+    make clean && make && make install && cd .. && \
+    rm -rf openssl-1.0.2g*
+
+ENV OPENSSL=/usr/local/openssl-1.0.2g/bin/openssl
+ENV PATH=/usr/local/openssl-1.0.2g/bin:$PATH
+
+# Install openssl 1.0.1j - "legacy" version
+RUN wget -q https://www.openssl.org/source/old/1.0.1/openssl-1.0.1j.tar.gz && \
+    tar -zxf openssl-1.0.1j.tar.gz && cd openssl-1.0.1j && \
+    ./config --openssldir=/usr/local/openssl-1.0.1j && \
+    make clean && make && make install && cd .. && \
+    rm -rf openssl-1.0.1j*
+
+ENV OPENSSL_LEGACY=/usr/local/openssl-1.0.1j/bin/openssl
+
+# Install openssl 1.1.1a - "next" version
+RUN wget -q https://www.openssl.org/source/openssl-1.1.1a.tar.gz && \
+    tar -zxf openssl-1.1.1a.tar.gz && cd openssl-1.1.1a && \
+    ./config --prefix=/usr/local/openssl-1.1.1a -Wl,--enable-new-dtags,-rpath,'$(LIBRPATH)' enable-ssl-trace && \
+    make clean && make && make install && cd .. && \
+    rm -rf openssl-1.1.1a*
+
+ENV OPENSSL_NEXT=/usr/local/openssl-1.1.1a/bin/openssl
+
+# Install Gnu TLS 3.4.10 (nettle 3.1) - main version, in the PATH
+RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.1.tar.gz && \
+    tar -zxf nettle-3.1.tar.gz && cd nettle-3.1 && \
+    ./configure --prefix=/usr/local/libnettle-3.1 --exec_prefix=/usr/local/libnettle-3.1  --disable-shared --disable-openssl && \
+    make && make install && cd .. && rm -rf nettle-3.1* && \
+    export PKG_CONFIG_PATH=/usr/local/libnettle-3.1/lib/pkgconfig:/usr/local/libnettle-3.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \
+    wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.4/gnutls-3.4.10.tar.xz && \
+    tar -xJf gnutls-3.4.10.tar.xz && cd gnutls-3.4.10 && \
+    ./configure --prefix=/usr/local/gnutls-3.4.10 --exec_prefix=/usr/local/gnutls-3.4.10 --disable-shared --without-p11-kit && \
+    make && make install && cat config.log && cd .. && \
+    rm -rf gnutls-3.4.10*
+
+ENV GNUTLS_CLI=/usr/local/gnutls-3.4.10/bin/gnutls-cli
+ENV GNUTLS_SERV=/usr/local/gnutls-3.4.10/bin/gnutls-serv
+ENV PATH=/usr/local/gnutls-3.4.10/bin:$PATH
+
+# Install Gnu TLS 3.3.8 (nettle 2.7) - "legacy" version
+RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz && \
+    tar -zxf nettle-2.7.1.tar.gz && cd nettle-2.7.1 && \
+    ./configure --prefix=/usr/local/libnettle-2.7.1 --exec_prefix=/usr/local/libnettle-2.7.1  --disable-shared --disable-openssl && \
+    make && make install && cd .. && rm -rf nettle-2.7.1* && \
+    export PKG_CONFIG_PATH=/usr/local/libnettle-2.7.1/lib/pkgconfig:/usr/local/libnettle-2.7.1/lib64/pkgconfig:/usr/local/lib/pkgconfig && \
+    wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.3/gnutls-3.3.8.tar.xz && \
+    tar -xJf gnutls-3.3.8.tar.xz && cd gnutls-3.3.8 && \
+    ./configure --prefix=/usr/local/gnutls-3.3.8 --exec_prefix=/usr/local/gnutls-3.3.8 --disable-shared --without-p11-kit && \
+    make && make install && cat config.log && cd .. && \
+    rm -rf gnutls-3.3.8*
+
+ENV GNUTLS_LEGACY_CLI=/usr/local/gnutls-3.3.8/bin/gnutls-cli
+ENV GNUTLS_LEGACY_SERV=/usr/local/gnutls-3.3.8/bin/gnutls-serv
+
+# Instal GNU TLS 3.7.2 (nettle 3.7) - "next" version
+RUN wget -q https://ftp.gnu.org/gnu/nettle/nettle-3.7.3.tar.gz && \
+    tar -zxf nettle-3.7.3.tar.gz && cd nettle-3.7.3 && \
+    ./configure --prefix=/usr/local/libnettle-3.7.3 --exec_prefix=/usr/local/libnettle-3.7.3  --disable-shared --disable-openssl && \
+    make && make install && cd .. && rm -rf nettle-3.7.3* && \
+    export PKG_CONFIG_PATH=/usr/local/libnettle-3.7.3/lib/pkgconfig:/usr/local/libnettle-3.7.3/lib64/pkgconfig:/usr/local/lib/pkgconfig && \
+    wget -q https://www.gnupg.org/ftp/gcrypt/gnutls/v3.7/gnutls-3.7.2.tar.xz && \
+    tar -xJf gnutls-3.7.2.tar.xz && cd gnutls-3.7.2 && \
+    ./configure --prefix=/usr/local/gnutls-3.7.2 --exec_prefix=/usr/local/gnutls-3.7.2 --disable-shared --without-p11-kit && \
+    make && make install && cat config.log && cd .. && \
+    rm -rf gnutls-3.7.2*
+
+ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli
+ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv
+
+# Install Python pip packages
+#
+# The pip wrapper scripts can get out of sync with pip due to upgrading it
+# outside the package manager, so invoke the module directly.
+#
+# Piping to cat suppresses the progress bar, but means that a failure
+# won't be caught (`stuff | cat` succeeds if cat succeeds, even if `stuff`
+# fails). The subsequent use of "pip config" (which requires pip >=10)
+# will however fail if the installation of a more recent pip failed.
+RUN python3 -m pip install pip --upgrade | cat && \
+    python3 -m pip config set global.progress_bar off && \
+    python3 -m pip install setuptools --upgrade && \
+    # For pylint we want a known version, as later versions may add checks at
+    # any time, making CI results unpredictable.
+    python3 -m pip install pylint==2.4.4 && \
+    # For mypy, use the earliest version that works with our code base.
+    # See https://github.com/ARMmbed/mbedtls/pull/3953 .
+    python3 -m pip install mypy==0.780 && \
+    true
+
+# Set locale for ARMCC to work
+RUN locale && \
+    locale-gen "en_US.UTF-8" && \
+    dpkg-reconfigure locales
+
+# Add user
+RUN useradd -m user
+
+# Create workspace
+ARG AGENT_WORKDIR=/var/lib/builds
+RUN mkdir -p ${AGENT_WORKDIR} && chown user:user ${AGENT_WORKDIR}
+USER user
+ENV AGENT_WORKDIR=${AGENT_WORKDIR}
+
+WORKDIR ${AGENT_WORKDIR}
+
+ENTRYPOINT ["bash"]
+

vars/common.groovy

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2019-2021, Arm Limited, All Rights Reserved
+ *  Copyright (c) 2019-2022, Arm Limited, All Rights Reserved
  *  SPDX-License-Identifier: Apache-2.0
  *
  *  Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -41,7 +41,7 @@ import groovy.transform.Field
 @Field docker_ecr = "666618195821.dkr.ecr.eu-west-1.amazonaws.com"
 @Field docker_repo = "$docker_ecr/$docker_repo_name"
 
-@Field linux_platforms = ["ubuntu-16.04", "ubuntu-18.04"]
+@Field linux_platforms = ["ubuntu-16.04", "ubuntu-18.04", "ubuntu-20.04"]
 @Field bsd_platforms = ["freebsd"]
 @Field bsd_compilers = ["clang"]
 @Field all_compilers = ['gcc', 'clang']

vars/dockerfile-builder-Jenkinsfile

@@ -9,6 +9,7 @@
  * Build options
  *  - DOCKER_IMAGE_16_04_ACTION
  *  - DOCKER_IMAGE_18_04_ACTION
+ *  - DOCKER_IMAGE_20_04_ACTION
  *
  * Environment variables:
  *  - GIT_CREDENTIALS_ID

vars/dockerfile_builder.groovy

@@ -14,6 +14,7 @@ def run_job() {
         stage('dockerfile-builder') {
             def jobs = gen_job_for_action(DOCKER_IMAGE_16_04_ACTION, 'ubuntu-16.04')
             jobs += gen_job_for_action(DOCKER_IMAGE_18_04_ACTION, 'ubuntu-18.04')
+            jobs += gen_job_for_action(DOCKER_IMAGE_20_04_ACTION, 'ubuntu-20.04')
             jobs.failFast = false
             parallel jobs
         }

vars/gen_jobs.groovy

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2019-2021, Arm Limited, All Rights Reserved
+ *  Copyright (c) 2019-2022, Arm Limited, All Rights Reserved
  *  SPDX-License-Identifier: Apache-2.0
  *
  *  Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -220,16 +220,10 @@ scripts/min_requirements.py --user
                 }
                 dir('src') {
                     checkout_repo.checkout_repo()
-                    /* ARMLMD_LICENSE_FILE is supposed to be set in the
-                     * Dockerfile, but the value there is out-of-date and we
-                     * have trouble re-building the images (due to the death
-                     * of Python 2), so we override it here as a temporary
-                     * work-around. */
                     writeFile file: 'steps.sh', text: """\
 #!/bin/sh
 set -eux
 ulimit -f 20971520
-export ARMLMD_LICENSE_FILE="[email protected]:[email protected]:[email protected]:[email protected]"
 export MBEDTLS_TEST_OUTCOME_FILE='${job_name}-outcome.csv'
 ${extra_setup_code}
 ./tests/scripts/all.sh --seed 4 --keep-going $component
@@ -528,12 +522,9 @@ pip install -r requirements.txt
                             if (example == 'atecc608a') {
                                 tag_filter = "--tag-filters HAS_CRYPTOKIT"
                             }
-                            /* See gen_all_sh_jobs() regarding setting of
-                             * ARMLMD_LICENSE_FILE here. */
                             sh """\
 ulimit -f 20971520
 . $WORKSPACE/mbed-venv/bin/activate
-export ARMLMD_LICENSE_FILE="[email protected]:[email protected]:[email protected]:[email protected]"
 mbed compile -m ${platform} -t ${compiler}
 """
                             for (int attempt = 1; attempt <= 3; attempt++) {