From 4938e5ae7d6331d20e04f890e6efe9983c34b7bd Mon Sep 17 00:00:00 2001 From: Vadim Markovtsev Date: Wed, 16 Nov 2022 11:38:52 +0100 Subject: [PATCH] [DEV-5385] Build CPython using BOLT Using the patch adapted from https://github.com/python/cpython/pull/95908 --- Dockerfile | 13 +++-- patches/cpython_configure_ac.patch | 80 ++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 patches/cpython_configure_ac.patch diff --git a/Dockerfile b/Dockerfile index f84509ff55..d7843cb5d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN apt-get update && \ # matches our production except -march=haswell, we have to downgrade -march because of GHA ENV OPT="-pipe -fno-semantic-interposition -march=haswell -mabm -maes -mno-pku -mno-sgx --param l1-cache-line-size=64 --param l1-cache-size=32 --param l2-cache-size=33792" -# Bolt: -Wl,--emit-relocs -fno-reorder-blocks-and-partition +ADD patches/cpython_configure_ac.patch / # runtime environment RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/etc/apt/sources.list && \ @@ -61,18 +61,25 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et python3-distutils html2text libjs-sphinxdoc && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen && \ + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" >>/etc/apt/sources.list.d/llvm.list && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-get update && \ + apt-get install -y bolt-15 && \ add-apt-repository -s ppa:deadsnakes/ppa && \ mkdir /cpython && \ cd /cpython && \ apt-get source python$PYTHON_VERSION && \ apt-get -s build-dep python$PYTHON_VERSION | grep "Inst " | cut -d" " -f2 | sort | tr '\n' ' ' >build_bloat && \ DEBIAN_FRONTEND="noninteractive" TZ="Europe/Madrid" apt-get build-dep -y python$PYTHON_VERSION && \ - rm /etc/apt/sources.list.d/deadsnakes* && \ + rm /etc/apt/sources.list.d/deadsnakes* /etc/apt/sources.list.d/llvm.list && \ cd python$PYTHON_VERSION* && \ sed -i 's/__main__/__skip__/g' Tools/scripts/run_tests.py && \ dch --bin-nmu -Dunstable "Optimized build" && \ echo 11 >debian/compat && \ sed -i 's/debhelper (>= 9)/debhelper (>= 11)/g' debian/control.in && \ + patch configure.ac >/et python$PYTHON_VERSION-venv* \ python$PYTHON_VERSION-full* && \ echo "========" && ls && \ - apt-get purge -y dpkg-dev devscripts software-properties-common html2text $(cat build_bloat) && \ + apt-get purge -y dpkg-dev devscripts software-properties-common html2text bolt-15 $(cat build_bloat) && \ apt-get autoremove -y && \ dpkg -i *python3.11*.deb && \ dpkg -i python3-minimal*.deb libpython3-stdlib*.deb && \ diff --git a/patches/cpython_configure_ac.patch b/patches/cpython_configure_ac.patch new file mode 100644 index 0000000000..9c602f968a --- /dev/null +++ b/patches/cpython_configure_ac.patch @@ -0,0 +1,80 @@ +--- configure.ac 2022-11-16 11:37:19.773231374 +0100 ++++ configure.ac.bolt 2022-11-16 11:37:11.161182410 +0100 +@@ -1751,7 +1751,16 @@ + # non-suffixed name in their versioned llvm directory. + + llvm_bin_dir='' +-llvm_path="${PATH}" ++ ++AC_ARG_WITH(llvm, ++ AS_HELP_STRING([--with-llvm=PATH], ++ [specify the directory where LLVM binaries are installed]), ++[ ++ llvm_path=${withval} ++],[ ++ llvm_path="${PATH}" ++]) ++ + if test "${CC}" = "clang" + then + clang_bin=`which clang` +@@ -1889,6 +1898,59 @@ + LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" + fi + ++# Enable bolt flags ++Py_BOLT='false' ++AC_MSG_CHECKING(for --enable-bolt) ++AC_ARG_ENABLE(bolt, AS_HELP_STRING( ++ [--enable-bolt], ++ [enable usage of the llvm-bolt post-link optimizer (default is no)]), ++[ ++if test "$enableval" != no ++then ++ Py_BOLT='true' ++ AC_MSG_RESULT(yes); ++else ++ Py_BOLT='false' ++ AC_MSG_RESULT(no); ++fi], ++[AC_MSG_RESULT(no)]) ++ ++AC_SUBST(PREBOLT_RULE) ++if test "$Py_BOLT" = 'true' ; then ++ PREBOLT_RULE="${DEF_MAKE_ALL_RULE}" ++ DEF_MAKE_ALL_RULE="bolt-opt" ++ DEF_MAKE_RULE="build_all" ++ ++ # These flags are required for bolt to work: ++ CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" ++ LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs" ++ ++ # These flags are required to get good performance from bolt: ++ CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie" ++ # We want to add these no-pie flags to linking executables but not shared libraries: ++ LINKCC="$LINKCC -fno-pie -no-pie" ++ # Designate the DWARF version into 4 since the LLVM-BOLT does not support DWARF5 yet. ++ CFLAGS="$CFLAGS -gdwarf-4" ++ LDFLAGS="$LDFLAGS -gdwarf-4" ++ AC_SUBST(LLVM_BOLT) ++ AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path}) ++ if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}" ++ then ++ AC_MSG_RESULT("Found llvm-bolt") ++ else ++ AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.]) ++ fi ++ ++ AC_SUBST(MERGE_FDATA) ++ AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path}) ++ if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}" ++ then ++ AC_MSG_RESULT("Found merge-fdata") ++ else ++ AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.]) ++ fi ++fi ++ + # Enable PGO flags. + AC_SUBST(PGO_PROF_GEN_FLAG) + AC_SUBST(PGO_PROF_USE_FLAG)