[DEV-5385] Build CPython using BOLT

Dockerfile

@@ -48,7 +48,7 @@ RUN apt-get update && \
 
 # matches our production except -march=haswell, we have to downgrade -march because of GHA
 ENV OPT="-pipe -fno-semantic-interposition -march=haswell -mabm -maes -mno-pku -mno-sgx --param l1-cache-line-size=64 --param l1-cache-size=32 --param l2-cache-size=33792"
-# Bolt: -Wl,--emit-relocs -fno-reorder-blocks-and-partition
+ADD patches/cpython_configure_ac.patch patches/cpython_makefile.patch patches/cpython_rules.patch /
 
 # runtime environment
 RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/etc/apt/sources.list && \
@@ -61,18 +61,27 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et
       python3-distutils html2text libjs-sphinxdoc && \
     echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen && \
+    echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main" >>/etc/apt/sources.list.d/llvm.list && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt-get update && \
+    apt-get install -y bolt-16 llvm-16 && \
+    export LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-16/bin/llvm-symbolizer && \
     add-apt-repository -s ppa:deadsnakes/ppa && \
     mkdir /cpython && \
     cd /cpython && \
     apt-get source python$PYTHON_VERSION && \
     apt-get -s build-dep python$PYTHON_VERSION | grep "Inst " | cut -d" " -f2 | sort | tr '\n' ' ' >build_bloat && \
     DEBIAN_FRONTEND="noninteractive" TZ="Europe/Madrid" apt-get build-dep -y python$PYTHON_VERSION && \
-    rm /etc/apt/sources.list.d/deadsnakes* && \
+    rm /etc/apt/sources.list.d/deadsnakes* /etc/apt/sources.list.d/llvm.list && \
     cd python$PYTHON_VERSION* && \
     sed -i 's/__main__/__skip__/g' Tools/scripts/run_tests.py && \
     dch --bin-nmu -Dunstable "Optimized build" && \
     echo 11 >debian/compat && \
     sed -i 's/debhelper (>= 9)/debhelper (>= 11)/g' debian/control.in && \
+    patch configure.ac </cpython_configure_ac.patch && \
+    patch Makefile.pre.in </cpython_makefile.patch && \
+    patch debian/rules </cpython_rules.patch && \
+    rm /cpython_configure_ac.patch /cpython_makefile.patch /cpython_rules.patch && \
     DEB_CFLAGS_SET="$OPT" DEB_LDFLAGS_SET="$OPT" dpkg-buildpackage -uc -b -j$(getconf _NPROCESSORS_ONLN) && \
     cd .. && \
 	apt-get source python3 && \
@@ -101,7 +110,7 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et
       python$PYTHON_VERSION-venv* \
       python$PYTHON_VERSION-full* && \
     echo "========" && ls && \
-    apt-get purge -y dpkg-dev devscripts software-properties-common html2text $(cat build_bloat) && \
+    apt-get purge -y dpkg-dev devscripts software-properties-common html2text bolt-16 llvm-16 $(cat build_bloat) && \
     apt-get autoremove -y && \
     dpkg -i *python3.11*.deb && \
     dpkg -i python3-minimal*.deb libpython3-stdlib*.deb && \

patches/cpython_configure_ac.patch

@@ -0,0 +1,80 @@
+--- configure.ac	2022-11-16 11:37:19.773231374 +0100
++++ configure.ac.bolt	2022-11-16 11:37:11.161182410 +0100
+@@ -1751,7 +1751,16 @@
+ # non-suffixed name in their versioned llvm directory.
+
+ llvm_bin_dir=''
+-llvm_path="${PATH}"
++
++AC_ARG_WITH(llvm,
++            AS_HELP_STRING([--with-llvm=PATH],
++                           [specify the directory where LLVM binaries are installed]),
++[
++    llvm_path=${withval}
++],[
++    llvm_path="${PATH}"
++])
++
+ if test "${CC}" = "clang"
+ then
+   clang_bin=`which clang`
+@@ -1889,6 +1898,59 @@
+   LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS"
+ fi
+
++# Enable bolt flags
++Py_BOLT='false'
++AC_MSG_CHECKING(for --enable-bolt)
++AC_ARG_ENABLE(bolt, AS_HELP_STRING(
++                [--enable-bolt],
++                [enable usage of the llvm-bolt post-link optimizer (default is no)]),
++[
++if test "$enableval" != no
++then
++  Py_BOLT='true'
++  AC_MSG_RESULT(yes);
++else
++  Py_BOLT='false'
++  AC_MSG_RESULT(no);
++fi],
++[AC_MSG_RESULT(no)])
++
++AC_SUBST(PREBOLT_RULE)
++if test "$Py_BOLT" = 'true' ; then
++  PREBOLT_RULE="${DEF_MAKE_ALL_RULE}"
++  DEF_MAKE_ALL_RULE="bolt-opt"
++  DEF_MAKE_RULE="build_all"
++
++  # These flags are required for bolt to work:
++  CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition"
++  LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs"
++
++  # These flags are required to get good performance from bolt:
++  CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie"
++  # We want to add these no-pie flags to linking executables but not shared libraries:
++  LINKCC="$LINKCC -fno-pie -no-pie"
++  # Designate the DWARF version into 4 since the LLVM-BOLT does not support DWARF5 yet.
++  CFLAGS="$CFLAGS -gdwarf-4"
++  LDFLAGS="$LDFLAGS -gdwarf-4"
++  AC_SUBST(LLVM_BOLT)
++  AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path})
++  if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}"
++  then
++    AC_MSG_RESULT("Found llvm-bolt")
++  else
++    AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.])
++  fi
++
++  AC_SUBST(MERGE_FDATA)
++  AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path})
++  if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}"
++  then
++    AC_MSG_RESULT("Found merge-fdata")
++  else
++    AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.])
++  fi
++fi
++
+ # Enable PGO flags.
+ AC_SUBST(PGO_PROF_GEN_FLAG)
+ AC_SUBST(PGO_PROF_USE_FLAG)

patches/cpython_makefile.patch

@@ -0,0 +1,19 @@
+--- Makefile.pre.in	2022-11-09 19:45:00.000000000 +0100
++++ Makefile.pre.in.bolt	2022-11-16 12:06:59.565683052 +0100
+@@ -659,6 +659,16 @@
+ 	-rm -f profile-clean-stamp
+ 	$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)"
+
++bolt-opt: @PREBOLT_RULE@
++	rm -f *.fdata
++	@LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst
++	./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true
++	@MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata
++	@LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=all -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot
++	rm -f *.fdata
++	rm -f $(BUILDPYTHON).bolt_inst
++	mv $(BUILDPYTHON).bolt $(BUILDPYTHON)
++
+ # Compile and run with gcov
+ .PHONY=coverage coverage-lcov coverage-report
+ coverage:

patches/cpython_rules.patch

@@ -0,0 +1,12 @@
+--- rules	2022-11-16 12:34:58.128078268 +0100
++++ rules.bolt	2022-11-16 12:37:40.129579671 +0100
+@@ -394,7 +394,8 @@
+ 	  CPPFLAGS="$(DPKG_CPPFLAGS)" LDFLAGS="$(DPKG_LDFLAGS) $(LTO_CFLAGS)" \
+ 	    $(config_site) \
+ 	    ../configure \
+-		$(common_configure_args)
++		$(common_configure_args) \
++		--with-llvm=/usr/lib/llvm-16/bin --enable-bolt
+
+ 	$(call __post_configure,$(buildd_static))
+ 	touch $@