[1/4] docker: Add Hexagon image

From: Alessandro Di Federico <ale@rev.ng>

Signed-off-by: Alessandro Di Federico <ale@rev.ng>
[PMD: Base on qemu/debian10, add missing EXTRA_FILES, remove X86]
Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 tests/docker/Makefile.include                 |   2 +
 .../dockerfiles/debian-hexagon-cross.docker   |  23 +++
 .../build-toolchain.sh                        | 141 ++++++++++++++++++
 3 files changed, 166 insertions(+)
 create mode 100644 tests/docker/dockerfiles/debian-hexagon-cross.docker
 create mode 100755 tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh

Reviewed-by: Brian Cain <bcain@quicinc.com>

> -----Original Message-----
> From: Philippe Mathieu-Daudé <philippe.mathieu.daude@gmail.com> On
> Behalf Of Philippe Mathieu-Daudé
> Sent: Sunday, February 28, 2021 4:23 PM
> To: qemu-devel@nongnu.org
> Cc: Fam Zheng <fam@euphon.net>; Taylor Simpson
> <tsimpson@quicinc.com>; Philippe Mathieu-Daudé <philmd@redhat.com>;
> Alex Bennée <alex.bennee@linaro.org>; Alessandro Di Federico
> <ale.qemu@rev.ng>; Brian Cain <bcain@quicinc.com>; Alessandro Di
> Federico <ale@rev.ng>; Philippe Mathieu-Daudé <f4bug@amsat.org>
> Subject: [EXT] [PATCH 1/4] docker: Add Hexagon image
>
> From: Alessandro Di Federico <ale@rev.ng>
>
> Signed-off-by: Alessandro Di Federico <ale@rev.ng>
> [PMD: Base on qemu/debian10, add missing EXTRA_FILES, remove X86]
> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
> ---
>  tests/docker/Makefile.include                 |   2 +
>  .../dockerfiles/debian-hexagon-cross.docker   |  23 +++
>  .../build-toolchain.sh                        | 141 ++++++++++++++++++
>  3 files changed, 166 insertions(+)
>  create mode 100644 tests/docker/dockerfiles/debian-hexagon-cross.docker
>  create mode 100755 tests/docker/dockerfiles/debian-hexagon-
> cross.docker.d/build-toolchain.sh
>
> diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
> index 7cab761bf5b..3aee031c4e9 100644
> --- a/tests/docker/Makefile.include
> +++ b/tests/docker/Makefile.include
> @@ -144,6 +144,8 @@ docker-image-debian-riscv64-cross: docker-image-
> debian10
>  docker-image-debian-s390x-cross: docker-image-debian10
>  docker-image-debian-sh4-cross: docker-image-debian10
>  docker-image-debian-sparc64-cross: docker-image-debian10
> +docker-image-debian-hexagon-cross: \
> +
> +EXTRA_FILES:=$(SRC_PATH)/tests/docker/dockerfiles/debian-hexagon-
> cross.
> +docker.d/build-toolchain.sh
>
>  # Specialist build images, sometimes very limited tools
>  docker-image-debian-tricore-cross: docker-image-debian10 diff --git
> a/tests/docker/dockerfiles/debian-hexagon-cross.docker
> b/tests/docker/dockerfiles/debian-hexagon-cross.docker
> new file mode 100644
> index 00000000000..7aa28767311
> --- /dev/null
> +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker
> @@ -0,0 +1,23 @@
> +FROM qemu/debian10
> +
> +RUN apt update && \
> +    DEBIAN_FRONTEND=noninteractive eatmydata \
> +    apt install -y --no-install-recommends \
> +        bison \
> +        cmake \
> +        flex \
> +        lld \
> +        rsync \
> +        wget
> +
> +ENV TOOLCHAIN_INSTALL /usr
> +ENV ROOTFS /usr
> +
> +ENV LLVM_URL
> +https://github.com/llvm/llvm-
> project/archive/3d8149c2a1228609fd7d7c91a0
> +4681304a2f0ca9.tar.gz ENV MUSL_URL
> +https://github.com/quic/musl/archive/aff74b395fbf59cd7e93b3691905aa1
> af6
> +c0778c.tar.gz ENV LINUX_URL
> +https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.6.18.tar.xz
> +
> +ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh
> +
> +RUN ( cd /root/hexagon-toolchain && ./build-toolchain.sh ) && \
> +    rm -rf /root/hexagon-toolchain
> diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-
> toolchain.sh b/tests/docker/dockerfiles/debian-hexagon-
> cross.docker.d/build-toolchain.sh
> new file mode 100755
> index 00000000000..19b1c9f83e1
> --- /dev/null
> +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolc
> +++ hain.sh
> @@ -0,0 +1,141 @@
> +#!/bin/bash
> +
> +set -e
> +
> +BASE=$(readlink -f ${PWD})
> +
> +TOOLCHAIN_INSTALL=$(readlink -f "$TOOLCHAIN_INSTALL")
> ROOTFS=$(readlink
> +-f "$ROOTFS")
> +
> +TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin
> +HEX_SYSROOT=${TOOLCHAIN_INSTALL}/hexagon-unknown-linux-musl
> +HEX_TOOLS_TARGET_BASE=${HEX_SYSROOT}/usr
> +
> +function cdp() {
> +  DIR="$1"
> +  mkdir -p "$DIR"
> +  cd "$DIR"
> +}
> +
> +function fetch() {
> +  DIR="$1"
> +  URL="$2"
> +  TEMP="$(readlink -f "$PWD/tmp.tar.gz")"
> +  wget --quiet "$URL" -O "$TEMP"
> +  cdp "$DIR"
> +  tar xaf "$TEMP" --strip-components=1
> +  rm "$TEMP"
> +  cd -
> +}
> +
> +build_llvm_clang() {
> +  fetch "$BASE/llvm-project" "$LLVM_URL"
> +  cdp "$BASE/build-llvm"
> +
> +  cmake -G Ninja \
> +    -DCMAKE_BUILD_TYPE=Release \
> +    -DCMAKE_INSTALL_PREFIX=${TOOLCHAIN_INSTALL} \
> +    -DLLVM_ENABLE_LLD=ON \
> +    -DLLVM_TARGETS_TO_BUILD="Hexagon" \
> +    -DLLVM_ENABLE_PROJECTS="clang;lld" \
> +    "$BASE/llvm-project/llvm"
> +  ninja all install
> +  cd ${TOOLCHAIN_BIN}
> +  ln -sf clang hexagon-unknown-linux-musl-clang  ln -sf clang++
> + hexagon-unknown-linux-musl-clang++
> +  ln -sf llvm-ar hexagon-unknown-linux-musl-ar  ln -sf llvm-objdump
> + hexagon-unknown-linux-musl-objdump
> +  ln -sf llvm-objcopy hexagon-unknown-linux-musl-objcopy
> +  ln -sf llvm-readelf hexagon-unknown-linux-musl-readelf
> +  ln -sf llvm-ranlib hexagon-unknown-linux-musl-ranlib
> +
> +  # workaround for now:
> +  cat <<EOF > hexagon-unknown-linux-musl.cfg
> +-G0 --sysroot=${HEX_SYSROOT}
> +EOF
> +}
> +
> +build_clang_rt() {
> +  cdp "$BASE/build-clang_rt"
> +  cmake -G Ninja \
> +    -DCMAKE_BUILD_TYPE=Release \
> +    -DLLVM_CONFIG_PATH="$BASE/build-llvm/bin/llvm-config" \
> +    -DCMAKE_ASM_FLAGS="-G0 -mlong-calls -fno-pic --target=hexagon-
> unknown-linux-musl " \
> +    -DCMAKE_SYSTEM_NAME=Linux \
> +    -DCMAKE_C_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-
> musl-clang" \
> +    -DCMAKE_ASM_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-
> linux-musl-clang" \
> +    -DCMAKE_INSTALL_PREFIX=${HEX_TOOLS_TARGET_BASE} \
> +    -DCMAKE_CROSSCOMPILING=ON \
> +    -DCMAKE_C_COMPILER_FORCED=ON \
> +    -DCMAKE_CXX_COMPILER_FORCED=ON \
> +    -DCOMPILER_RT_BUILD_BUILTINS=ON \
> +    -DCOMPILER_RT_BUILTINS_ENABLE_PIC=OFF \
> +    -DCMAKE_SIZEOF_VOID_P=4 \
> +    -DCOMPILER_RT_OS_DIR= \
> +    -DCAN_TARGET_hexagon=1 \
> +    -DCAN_TARGET_x86_64=0 \
> +    -DCOMPILER_RT_SUPPORTED_ARCH=hexagon \
> +    -DLLVM_ENABLE_PROJECTS="compiler-rt" \
> +    "$BASE/llvm-project/compiler-rt"
> +  ninja install-compiler-rt
> +}
> +
> +build_musl_headers() {
> +  fetch "$BASE/musl" "$MUSL_URL"
> +  cd "$BASE/musl"
> +  make clean
> +  CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
> +    CROSS_COMPILE=hexagon-unknown-linux-musl \
> +    LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
> +    CROSS_CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --
> target=hexagon-unknown-linux-musl" \
> +    ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
> +  PATH=${TOOLCHAIN_BIN}:$PATH make CROSS_COMPILE= install-headers
> +
> +  cd ${HEX_SYSROOT}/..
> +  ln -sf hexagon-unknown-linux-musl hexagon }
> +
> +build_kernel_headers() {
> +  fetch "$BASE/linux" "$LINUX_URL"
> +  mkdir -p "$BASE/build-linux"
> +  cd "$BASE/linux"
> +  make O=../build-linux ARCH=hexagon \
> +   KBUILD_CFLAGS_KERNEL="-mlong-calls" \
> +   CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
> +   LD=${TOOLCHAIN_BIN}/ld.lld \
> +   KBUILD_VERBOSE=1 comet_defconfig
> +  make mrproper
> +
> +  cd "$BASE/build-linux"
> +  make \
> +    ARCH=hexagon \
> +    CC=${TOOLCHAIN_BIN}/clang \
> +    INSTALL_HDR_PATH=${HEX_TOOLS_TARGET_BASE} \
> +    V=1 \
> +    headers_install
> +}
> +
> +build_musl() {
> +  cd "$BASE/musl"
> +  make clean
> +  CROSS_COMPILE=hexagon-unknown-linux-musl- \
> +    AR=llvm-ar \
> +    RANLIB=llvm-ranlib \
> +    STRIP=llvm-strip \
> +    CC=clang \
> +    LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
> +    CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --
> target=hexagon-unknown-linux-musl" \
> +    ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
> +  PATH=${TOOLCHAIN_BIN}/:$PATH make CROSS_COMPILE= install
> +  cd ${HEX_TOOLS_TARGET_BASE}/lib
> +  ln -sf libc.so ld-musl-hexagon.so
> +  ln -sf ld-musl-hexagon.so ld-musl-hexagon.so.1
> +  cdp ${HEX_TOOLS_TARGET_BASE}/../lib
> +  ln -sf ../usr/lib/ld-musl-hexagon.so.1 }
> +
> +build_llvm_clang
> +build_kernel_headers
> +build_musl_headers
> +build_clang_rt
> +build_musl
> --
> 2.26.2

Philippe Mathieu-Daudé <f4bug@amsat.org> writes:

> From: Alessandro Di Federico <ale@rev.ng>
>
> Signed-off-by: Alessandro Di Federico <ale@rev.ng>
> [PMD: Base on qemu/debian10, add missing EXTRA_FILES, remove X86]
> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
> ---
>  tests/docker/Makefile.include                 |   2 +
>  .../dockerfiles/debian-hexagon-cross.docker   |  23 +++
>  .../build-toolchain.sh                        | 141 ++++++++++++++++++
>  3 files changed, 166 insertions(+)
>  create mode 100644 tests/docker/dockerfiles/debian-hexagon-cross.docker
>  create mode 100755 tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh
>
> diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
> index 7cab761bf5b..3aee031c4e9 100644
> --- a/tests/docker/Makefile.include
> +++ b/tests/docker/Makefile.include
> @@ -144,6 +144,8 @@ docker-image-debian-riscv64-cross: docker-image-debian10
>  docker-image-debian-s390x-cross: docker-image-debian10
>  docker-image-debian-sh4-cross: docker-image-debian10
>  docker-image-debian-sparc64-cross: docker-image-debian10
> +docker-image-debian-hexagon-cross: \
> +	EXTRA_FILES:=$(SRC_PATH)/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh
>  
>  # Specialist build images, sometimes very limited tools
>  docker-image-debian-tricore-cross: docker-image-debian10
> diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker
> new file mode 100644
> index 00000000000..7aa28767311
> --- /dev/null
> +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker
> @@ -0,0 +1,23 @@
> +FROM qemu/debian10
> +
> +RUN apt update && \
> +    DEBIAN_FRONTEND=noninteractive eatmydata \
> +    apt install -y --no-install-recommends \
> +        bison \
> +        cmake \
> +        flex \
> +        lld \
> +        rsync \
> +        wget
> +
> +ENV TOOLCHAIN_INSTALL /usr
> +ENV ROOTFS /usr
> +
> +ENV LLVM_URL https://github.com/llvm/llvm-project/archive/3d8149c2a1228609fd7d7c91a04681304a2f0ca9.tar.gz
> +ENV MUSL_URL https://github.com/quic/musl/archive/aff74b395fbf59cd7e93b3691905aa1af6c0778c.tar.gz
> +ENV LINUX_URL https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.6.18.tar.xz
> +
> +ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh
> +
> +RUN ( cd /root/hexagon-toolchain && ./build-toolchain.sh ) && \
> +    rm -rf /root/hexagon-toolchain

So this is attempting to everything in one step so we don't inflate the
layers too much?

I believe this is something multi-stage builds are meant to deal with:

  https://docs.docker.com/develop/develop-images/multistage-build/

but I've not gotten one to work in practice. For now I'm trying this out
on my beefy machine and I'll see if I have the permissions required to
upload it to gitlab.

> diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh b/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh
> new file mode 100755
> index 00000000000..19b1c9f83e1
> --- /dev/null
> +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh
> @@ -0,0 +1,141 @@
> +#!/bin/bash
> +
> +set -e
> +
> +BASE=$(readlink -f ${PWD})
> +
> +TOOLCHAIN_INSTALL=$(readlink -f "$TOOLCHAIN_INSTALL")
> +ROOTFS=$(readlink -f "$ROOTFS")
> +
> +TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin
> +HEX_SYSROOT=${TOOLCHAIN_INSTALL}/hexagon-unknown-linux-musl
> +HEX_TOOLS_TARGET_BASE=${HEX_SYSROOT}/usr
> +
> +function cdp() {
> +  DIR="$1"
> +  mkdir -p "$DIR"
> +  cd "$DIR"
> +}
> +
> +function fetch() {
> +  DIR="$1"
> +  URL="$2"
> +  TEMP="$(readlink -f "$PWD/tmp.tar.gz")"
> +  wget --quiet "$URL" -O "$TEMP"
> +  cdp "$DIR"
> +  tar xaf "$TEMP" --strip-components=1
> +  rm "$TEMP"
> +  cd -
> +}
> +
> +build_llvm_clang() {
> +  fetch "$BASE/llvm-project" "$LLVM_URL"
> +  cdp "$BASE/build-llvm"
> +
> +  cmake -G Ninja \
> +    -DCMAKE_BUILD_TYPE=Release \
> +    -DCMAKE_INSTALL_PREFIX=${TOOLCHAIN_INSTALL} \
> +    -DLLVM_ENABLE_LLD=ON \
> +    -DLLVM_TARGETS_TO_BUILD="Hexagon" \
> +    -DLLVM_ENABLE_PROJECTS="clang;lld" \
> +    "$BASE/llvm-project/llvm"
> +  ninja all install
> +  cd ${TOOLCHAIN_BIN}
> +  ln -sf clang hexagon-unknown-linux-musl-clang
> +  ln -sf clang++ hexagon-unknown-linux-musl-clang++
> +  ln -sf llvm-ar hexagon-unknown-linux-musl-ar
> +  ln -sf llvm-objdump hexagon-unknown-linux-musl-objdump
> +  ln -sf llvm-objcopy hexagon-unknown-linux-musl-objcopy
> +  ln -sf llvm-readelf hexagon-unknown-linux-musl-readelf
> +  ln -sf llvm-ranlib hexagon-unknown-linux-musl-ranlib
> +
> +  # workaround for now:
> +  cat <<EOF > hexagon-unknown-linux-musl.cfg
> +-G0 --sysroot=${HEX_SYSROOT}
> +EOF
> +}
> +
> +build_clang_rt() {
> +  cdp "$BASE/build-clang_rt"
> +  cmake -G Ninja \
> +    -DCMAKE_BUILD_TYPE=Release \
> +    -DLLVM_CONFIG_PATH="$BASE/build-llvm/bin/llvm-config" \
> +    -DCMAKE_ASM_FLAGS="-G0 -mlong-calls -fno-pic --target=hexagon-unknown-linux-musl " \
> +    -DCMAKE_SYSTEM_NAME=Linux \
> +    -DCMAKE_C_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \
> +    -DCMAKE_ASM_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \
> +    -DCMAKE_INSTALL_PREFIX=${HEX_TOOLS_TARGET_BASE} \
> +    -DCMAKE_CROSSCOMPILING=ON \
> +    -DCMAKE_C_COMPILER_FORCED=ON \
> +    -DCMAKE_CXX_COMPILER_FORCED=ON \
> +    -DCOMPILER_RT_BUILD_BUILTINS=ON \
> +    -DCOMPILER_RT_BUILTINS_ENABLE_PIC=OFF \
> +    -DCMAKE_SIZEOF_VOID_P=4 \
> +    -DCOMPILER_RT_OS_DIR= \
> +    -DCAN_TARGET_hexagon=1 \
> +    -DCAN_TARGET_x86_64=0 \
> +    -DCOMPILER_RT_SUPPORTED_ARCH=hexagon \
> +    -DLLVM_ENABLE_PROJECTS="compiler-rt" \
> +    "$BASE/llvm-project/compiler-rt"
> +  ninja install-compiler-rt
> +}
> +
> +build_musl_headers() {
> +  fetch "$BASE/musl" "$MUSL_URL"
> +  cd "$BASE/musl"
> +  make clean
> +  CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
> +    CROSS_COMPILE=hexagon-unknown-linux-musl \
> +    LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
> +    CROSS_CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \
> +    ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
> +  PATH=${TOOLCHAIN_BIN}:$PATH make CROSS_COMPILE= install-headers
> +
> +  cd ${HEX_SYSROOT}/..
> +  ln -sf hexagon-unknown-linux-musl hexagon
> +}
> +
> +build_kernel_headers() {
> +  fetch "$BASE/linux" "$LINUX_URL"
> +  mkdir -p "$BASE/build-linux"
> +  cd "$BASE/linux"
> +  make O=../build-linux ARCH=hexagon \
> +   KBUILD_CFLAGS_KERNEL="-mlong-calls" \
> +   CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
> +   LD=${TOOLCHAIN_BIN}/ld.lld \
> +   KBUILD_VERBOSE=1 comet_defconfig
> +  make mrproper
> +
> +  cd "$BASE/build-linux"
> +  make \
> +    ARCH=hexagon \
> +    CC=${TOOLCHAIN_BIN}/clang \
> +    INSTALL_HDR_PATH=${HEX_TOOLS_TARGET_BASE} \
> +    V=1 \
> +    headers_install
> +}
> +
> +build_musl() {
> +  cd "$BASE/musl"
> +  make clean
> +  CROSS_COMPILE=hexagon-unknown-linux-musl- \
> +    AR=llvm-ar \
> +    RANLIB=llvm-ranlib \
> +    STRIP=llvm-strip \
> +    CC=clang \
> +    LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
> +    CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \
> +    ./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
> +  PATH=${TOOLCHAIN_BIN}/:$PATH make CROSS_COMPILE= install
> +  cd ${HEX_TOOLS_TARGET_BASE}/lib
> +  ln -sf libc.so ld-musl-hexagon.so
> +  ln -sf ld-musl-hexagon.so ld-musl-hexagon.so.1
> +  cdp ${HEX_TOOLS_TARGET_BASE}/../lib
> +  ln -sf ../usr/lib/ld-musl-hexagon.so.1
> +}
> +
> +build_llvm_clang
> +build_kernel_headers
> +build_musl_headers
> +build_clang_rt
> +build_musl