Auto merge of rust-lang#103392 - RalfJung:miri, r=oli-obk

update Miri

I had to use a hacked version of josh to create this, so let's be careful with merging this and maybe wait a bit to see if the josh issue becomes more clear. But the history looks good to me, we are not adding duplicates of rustc commits that were previously mirrored to Miri.

Also I want to add some cross-testing of Miri in x.py.

Cargo.lock

@@ -2262,6 +2262,7 @@ dependencies = [
  "rand 0.8.5",
  "regex",
  "rustc-workspace-hack",
+ "rustc_version",
  "shell-escape",
  "smallvec",
  "ui_test",

src/bootstrap/test.rs

@@ -461,24 +461,30 @@ impl Step for RustDemangler {
 pub struct Miri {
     stage: u32,
     host: TargetSelection,
+    target: TargetSelection,
 }
 
 impl Step for Miri {
     type Output = ();
-    const ONLY_HOSTS: bool = true;
+    const ONLY_HOSTS: bool = false;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.path("src/tools/miri")
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Miri { stage: run.builder.top_stage, host: run.target });
+        run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
+            host: run.build_triple(),
+            target: run.target,
+        });
     }
 
     /// Runs `cargo test` for miri.
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let host = self.host;
+        let target = self.target;
         let compiler = builder.compiler(stage, host);
         // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
         // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
@@ -495,7 +501,7 @@ impl Step for Miri {
         builder.ensure(compile::Std::new(compiler_std, host));
         let sysroot = builder.sysroot(compiler_std);
 
-        // # Run `cargo miri setup`.
+        // # Run `cargo miri setup` for the given target.
         let mut cargo = tool::prepare_tool_cargo(
             builder,
             compiler,
@@ -508,6 +514,7 @@ impl Step for Miri {
         );
         cargo.add_rustc_lib_path(builder, compiler);
         cargo.arg("--").arg("miri").arg("setup");
+        cargo.arg("--target").arg(target.rustc_target_arg());
 
         // Tell `cargo miri setup` where to find the sources.
         cargo.env("MIRI_LIB_SRC", builder.src.join("library"));
@@ -556,17 +563,54 @@ impl Step for Miri {
         cargo.add_rustc_lib_path(builder, compiler);
 
         // miri tests need to know about the stage sysroot
-        cargo.env("MIRI_SYSROOT", miri_sysroot);
+        cargo.env("MIRI_SYSROOT", &miri_sysroot);
         cargo.env("MIRI_HOST_SYSROOT", sysroot);
         cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
-        cargo.env("MIRI", miri);
+        cargo.env("MIRI", &miri);
         // propagate --bless
         if builder.config.cmd.bless() {
             cargo.env("MIRI_BLESS", "Gesundheit");
         }
 
+        // Set the target.
+        cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg());
+        // Forward test filters.
+        cargo.arg("--").args(builder.config.cmd.test_args());
+
+        let mut cargo = Command::from(cargo);
+        builder.run(&mut cargo);
+
+        // # Run `cargo miri test`.
+        // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures
+        // that we get the desired output), but that is sufficient to make sure that the libtest harness
+        // itself executes properly under Miri.
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            host,
+            "run",
+            "src/tools/miri/cargo-miri",
+            SourceType::Submodule,
+            &[],
+        );
+        cargo.add_rustc_lib_path(builder, compiler);
+        cargo.arg("--").arg("miri").arg("test");
+        cargo
+            .arg("--manifest-path")
+            .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml"));
+        cargo.arg("--target").arg(target.rustc_target_arg());
+        cargo.arg("--tests"); // don't run doctests, they are too confused by the staging
         cargo.arg("--").args(builder.config.cmd.test_args());
 
+        // Tell `cargo miri` where to find things.
+        cargo.env("MIRI_SYSROOT", &miri_sysroot);
+        cargo.env("MIRI_HOST_SYSROOT", sysroot);
+        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+        cargo.env("MIRI", &miri);
+        // Debug things.
+        cargo.env("RUST_BACKTRACE", "1");
+
         let mut cargo = Command::from(cargo);
         builder.run(&mut cargo);
     }

src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh

@@ -25,3 +25,8 @@ python3 "$X_PY" test --stage 2 check-tools
 python3 "$X_PY" test --stage 2 src/tools/clippy
 python3 "$X_PY" test --stage 2 src/tools/rustfmt
 python3 "$X_PY" test --stage 2 src/tools/miri
+# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc.
+# Also cover some other targets (on both of these hosts) via cross-testing.
+python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc
+#FIXME(https://github.com/rust-lang/rust/issues/103519): macOS testing is currently disabled
+# python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin

src/tools/miri/.github/workflows/ci.yml

@@ -22,6 +22,7 @@ jobs:
       RUST_BACKTRACE: 1
       HOST_TARGET: ${{ matrix.host_target }}
     strategy:
+      fail-fast: false
       matrix:
         build: [linux64, macos, win32]
         include:
@@ -61,7 +62,7 @@ jobs:
           restore-keys: ${{ runner.os }}-cargo
 
       - name: Install rustup-toolchain-install-master
-        if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
         shell: bash
         run: |
           cargo install -f rustup-toolchain-install-master
@@ -89,11 +90,46 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Install required toolchain
-        # We need a toolchain that can actually build Miri, just a nightly won't do.
+
+      # This is exactly duplicated from above. GHA is pretty terrible when it comes
+      # to avoiding code duplication.
+
+      # Cache the global cargo directory, but NOT the local `target` directory which
+      # we cannot reuse anyway when the nightly changes (and it grows quite large
+      # over time).
+      - name: Add cache for cargo
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            # Taken from <https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci>.
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            # contains package information of crates installed via `cargo install`.
+            ~/.cargo/.crates.toml
+            ~/.cargo/.crates2.json
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo
+
+      - name: Install rustup-toolchain-install-master
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        shell: bash
+        run: |
+          cargo install -f rustup-toolchain-install-master
+
+      - name: Install "master" toolchain
+        shell: bash
         run: |
-          cargo install rustup-toolchain-install-master # TODO: cache this?
           ./rustup-toolchain "" -c clippy
+
+      - name: Show Rust version
+        run: |
+          rustup show
+          rustc -Vv
+          cargo -V
+
       - name: rustfmt
         run: ./miri fmt --check
       - name: clippy

src/tools/miri/CONTRIBUTING.md

@@ -104,7 +104,7 @@ MIRI_LOG=rustc_mir::interpret=info,miri::stacked_borrows ./miri run tests/pass/v
 In addition, you can set `MIRI_BACKTRACE=1` to get a backtrace of where an
 evaluation error was originally raised.
 
-#### UI testing
+### UI testing
 
 We use ui-testing in Miri, meaning we generate `.stderr` and `.stdout` files for the output
 produced by Miri. You can use `./miri bless` to automatically (re)generate these files when
@@ -257,7 +257,7 @@ Note: When you are working with a locally built rustc or any other toolchain tha
 is not the same as the one in `rust-version`, you should not have `.auto-everything` or
 `.auto-toolchain` as that will keep resetting your toolchain.
 
-```
+```sh
 rm -f .auto-everything .auto-toolchain
 ```
 
@@ -275,3 +275,51 @@ see <https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html>.
 
 With this, you should now have a working development setup! See
 [above](#building-and-testing-miri) for how to proceed working on Miri.
+
+## Advanced topic: Syncing with the rustc repo
+
+We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit
+changes between the rustc and Miri repositories. For now, a fork of josh needs to be built
+from source. This downloads and runs josh:
+
+```sh
+git clone https://github.com/RalfJung/josh
+cd josh
+cargo run --release -p josh-proxy -- --local=$(pwd)/local --remote=https://github.com --no-background
+```
+
+### Importing changes from the rustc repo
+
+We assume we start on an up-to-date master branch in the Miri repo.
+
+```sh
+# Fetch rustc side of the history. Takes ca 5 min the first time.
+# Do NOT change that commit ID, it needs to be exactly this!
+git fetch http://localhost:8000/rust-lang/rust.git:at_commit=75dd959a3a40eb5b4574f8d2e23aa6efbeb33573[:prefix=src/tools/miri]:/src/tools/miri.git master
+# Include that history into ours.
+git merge FETCH_HEAD -m "merge rustc history"
+# Update toolchain reference and apply formatting.
+./rustup-toolchain HEAD && ./miri fmt
+git commit -am "rustup"
+```
+
+Now push this to a new branch in your Miri fork, and create a PR. It is worth
+running `./miri test` locally in parallel, since the test suite in the Miri repo
+is stricter than the one on the rustc side, so some small tweaks might be
+needed.
+
+### Exporting changes to the rustc repo
+
+We will use the josh proxy to push to your fork of rustc. You need to make sure
+that the master branch of your fork is up-to-date. Also make sure that there
+exists no branch called `miri` in your fork. Then run the following in the Miri
+repo, assuming we are on an up-to-date master branch:
+
+```sh
+# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
+# Do NOT change that commit ID, it needs to be exactly this!
+git push http://localhost:8000/YOUR_NAME/rust.git:at_commit=75dd959a3a40eb5b4574f8d2e23aa6efbeb33573[:prefix=src/tools/miri]:/src/tools/miri.git -o base=master HEAD:miri
+```
+
+This will create a new branch in your fork, and the output should include a link
+to create a rustc PR that will integrate those changes into the main repository.

src/tools/miri/Cargo.lock

@@ -320,9 +320,9 @@ dependencies = [
 
 [[package]]
 name = "libffi-sys"
-version = "2.0.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab4106b7f09d7b87d021334d5618fac1dfcfb824d4c5fe111ff0074dfd242e15"
+checksum = "84e78d02e5a8eae9c24c38ce6e6026f80e16dff76adcdae4bc5c6c52c2de4a60"
 dependencies = [
  "cc",
 ]
@@ -419,6 +419,7 @@ dependencies = [
  "rand",
  "regex",
  "rustc-workspace-hack",
+ "rustc_version",
  "shell-escape",
  "smallvec",
  "ui_test",

src/tools/miri/Cargo.toml

@@ -31,14 +31,17 @@ smallvec = "1.7"
 rustc-workspace-hack = "1.0.0"
 measureme = "10.0.0"
 
-[target."cfg(unix)".dependencies]
+[target.'cfg(unix)'.dependencies]
 libc = "0.2"
+
+[target.'cfg(target_os = "linux")'.dependencies]
 libffi = "3.0.0"
 libloading = "0.7"
 
 [dev-dependencies]
 colored = "2"
 ui_test = "0.3.1"
+rustc_version = "0.4"
 # Features chosen to match those required by env_logger, to avoid rebuilds
 regex = { version = "1.5.5", default-features = false, features = ["perf", "std"] }
 lazy_static = "1.4.0"

src/tools/miri/README.md

@@ -377,6 +377,11 @@ to Miri failing to detect cases of undefined behavior in a program.
 * `-Zmiri-retag-fields` changes Stacked Borrows retagging to recurse into fields.
   This means that references in fields of structs/enums/tuples/arrays/... are retagged,
   and in particular, they are protected when passed as function arguments.
+* `-Zmiri-retag-fields=<all|none|scalar>` controls when Stacked Borrows retagging recurses into
+  fields. `all` means it always recurses (like `-Zmiri-retag-fields`), `none` means it never
+  recurses (the default), `scalar` means it only recurses for types where we would also emit
+  `noalias` annotations in the generated LLVM IR (types passed as indivudal scalars or pairs of
+  scalars).
 * `-Zmiri-tag-gc=<blocks>` configures how often the pointer tag garbage collector runs. The default
   is to search for and remove unreachable tags once every `10000` basic blocks. Setting this to
   `0` disables the garbage collector, which causes some programs to have explosive memory usage
@@ -435,11 +440,10 @@ Moreover, Miri recognizes some environment variables:
   purpose.
 * `MIRI_NO_STD` (recognized by `cargo miri` and the test suite) makes sure that the target's
   sysroot is built without libstd. This allows testing and running no_std programs.
-* `MIRI_BLESS` (recognized by the test suite) overwrite all `stderr` and `stdout` files
-  instead of checking whether the output matches.
-* `MIRI_SKIP_UI_CHECKS` (recognized by the test suite) don't check whether the
-  `stderr` or `stdout` files match the actual output. Useful for the rustc test suite
-  which has subtle differences that we don't care about.
+* `MIRI_BLESS` (recognized by the test suite and `cargo-miri-test/run-test.py`): overwrite all
+  `stderr` and `stdout` files instead of checking whether the output matches.
+* `MIRI_SKIP_UI_CHECKS` (recognized by the test suite): don't check whether the
+  `stderr` or `stdout` files match the actual output.
 
 The following environment variables are *internal* and must not be used by
 anyone but Miri itself. They are used to communicate between different Miri
@@ -532,6 +536,27 @@ extern "Rust" {
     /// This is internal and unstable and should not be used; we give it here
     /// just to be complete.
     fn miri_start_panic(payload: *mut u8) -> !;
+
+    /// Miri-provided extern function to get the internal unique identifier for the allocation that a pointer
+    /// points to. This is only useful as an input to `miri_print_stacks`, and it is a separate call because
+    /// getting a pointer to an allocation at runtime can change the borrow stacks in the allocation.
+    fn miri_get_alloc_id(ptr: *const ()) -> u64;
+
+    /// Miri-provided extern function to print (from the interpreter, not the program) the contents of all
+    /// borrow stacks in an allocation. The format of what this emits is unstable and may change at any time.
+    /// In particular, users should be aware that Miri will periodically attempt to garbage collect the
+    /// contents of all stacks. Callers of this function may wish to pass `-Zmiri-tag-gc=0` to disable the GC.
+    fn miri_print_stacks(alloc_id: u64);
+
+    /// Miri-provided extern function to print (from the interpreter, not the
+    /// program) the contents of a section of program memory, as bytes. Bytes
+    /// written using this function will emerge from the interpreter's stdout.
+    fn miri_write_to_stdout(bytes: &[u8]);
+
+    /// Miri-provided extern function to print (from the interpreter, not the
+    /// program) the contents of a section of program memory, as bytes. Bytes
+    /// written using this function will emerge from the interpreter's stderr.
+    fn miri_write_to_stderr(bytes: &[u8]);
 }
 ```
 

src/tools/miri/bench-cargo-miri/serde2/src/main.rs

@@ -5,6 +5,7 @@ use serde::Deserialize;
 use std::thread;
 
 #[derive(Deserialize)]
+#[allow(unused)]
 struct DeriveStruct {
     buffer: Vec<i16>,
 }

src/tools/miri/rust-version

@@ -1 +1 @@
-acb8934fd57b3c2740c4abac0a5728c2c9b1423b
+b1ab3b738ac718da74cd4aa0bb7f362d0adbdf84

src/tools/miri/src/bin/miri.rs

@@ -32,7 +32,7 @@ use rustc_middle::{
 };
 use rustc_session::{config::CrateType, search_paths::PathKind, CtfeBacktrace};
 
-use miri::{BacktraceStyle, ProvenanceMode};
+use miri::{BacktraceStyle, ProvenanceMode, RetagFields};
 
 struct MiriCompilerCalls {
     miri_config: miri::MiriConfig,
@@ -426,7 +426,14 @@ fn main() {
         } else if arg == "-Zmiri-mute-stdout-stderr" {
             miri_config.mute_stdout_stderr = true;
         } else if arg == "-Zmiri-retag-fields" {
-            miri_config.retag_fields = true;
+            miri_config.retag_fields = RetagFields::Yes;
+        } else if let Some(retag_fields) = arg.strip_prefix("-Zmiri-retag-fields=") {
+            miri_config.retag_fields = match retag_fields {
+                "all" => RetagFields::Yes,
+                "none" => RetagFields::No,
+                "scalar" => RetagFields::OnlyScalar,
+                _ => show_error!("`-Zmiri-retag-fields` can only be `all`, `none`, or `scalar`"),
+            };
         } else if arg == "-Zmiri-track-raw-pointers" {
             eprintln!(
                 "WARNING: `-Zmiri-track-raw-pointers` has no effect; it is enabled by default"