diff --git a/Cargo.toml b/Cargo.toml
index 92236aa0..9c7c7aa7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,7 +25,6 @@ compiler_builtins = { version = '0.1.2', optional = true }
 [dev-dependencies]
 trybuild = "1.0"
 rustversion = "1.0"
-walkdir = "2.3"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
diff --git a/tests/compile-fail/.gitignore b/tests/compile-fail/.gitignore
deleted file mode 100644
index 4dd9abc8..00000000
--- a/tests/compile-fail/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.stderr
diff --git a/tests/compile-fail/cfg/multi.stderr.beta b/tests/compile-fail/cfg/multi.stderr
similarity index 100%
rename from tests/compile-fail/cfg/multi.stderr.beta
rename to tests/compile-fail/cfg/multi.stderr
diff --git a/tests/compile-fail/impls/copy.stderr.beta b/tests/compile-fail/impls/copy.stderr
similarity index 100%
rename from tests/compile-fail/impls/copy.stderr.beta
rename to tests/compile-fail/impls/copy.stderr
diff --git a/tests/compile-fail/impls/eq.stderr.beta b/tests/compile-fail/impls/eq.stderr
similarity index 100%
rename from tests/compile-fail/impls/eq.stderr.beta
rename to tests/compile-fail/impls/eq.stderr
diff --git a/tests/compile-fail/non_integer_base/all_defined.stderr.beta b/tests/compile-fail/non_integer_base/all_defined.stderr
similarity index 100%
rename from tests/compile-fail/non_integer_base/all_defined.stderr.beta
rename to tests/compile-fail/non_integer_base/all_defined.stderr
diff --git a/tests/compile-fail/non_integer_base/all_missing.stderr.beta b/tests/compile-fail/non_integer_base/all_missing.stderr
similarity index 100%
rename from tests/compile-fail/non_integer_base/all_missing.stderr.beta
rename to tests/compile-fail/non_integer_base/all_missing.stderr
diff --git a/tests/compile-fail/trait/custom_impl.stderr.beta b/tests/compile-fail/trait/custom_impl.stderr
similarity index 100%
rename from tests/compile-fail/trait/custom_impl.stderr.beta
rename to tests/compile-fail/trait/custom_impl.stderr
diff --git a/tests/compile-fail/visibility/private_field.stderr.beta b/tests/compile-fail/visibility/private_field.stderr
similarity index 100%
rename from tests/compile-fail/visibility/private_field.stderr.beta
rename to tests/compile-fail/visibility/private_field.stderr
diff --git a/tests/compile-fail/visibility/private_flags.stderr.beta b/tests/compile-fail/visibility/private_flags.stderr
similarity index 100%
rename from tests/compile-fail/visibility/private_flags.stderr.beta
rename to tests/compile-fail/visibility/private_flags.stderr
diff --git a/tests/compile-fail/visibility/pub_const.stderr.beta b/tests/compile-fail/visibility/pub_const.stderr
similarity index 100%
rename from tests/compile-fail/visibility/pub_const.stderr.beta
rename to tests/compile-fail/visibility/pub_const.stderr
diff --git a/tests/compile.rs b/tests/compile.rs
index f3ee9122..7127acee 100644
--- a/tests/compile.rs
+++ b/tests/compile.rs
@@ -1,26 +1,12 @@
-use std::{
-    env,
-    fs,
-    ffi::OsStr,
-    io,
-    path::Path,
-};
-
-use walkdir::WalkDir;
-
-#[test]
+// Compiler messages change frequently enough that we can end up with
+// an impossible build between error messages emitted on various channels.
+// Since https://github.com/dtolnay/trybuild/pull/170 we always need to have a
+// `stderr` file for each test so we can't simply ignore the output on different channels.
+#[rustversion::attr(beta, test)]
+#[allow(dead_code)]
 fn fail() {
-    prepare_stderr_files("tests/compile-fail").unwrap();
-
     let t = trybuild::TestCases::new();
-
     t.compile_fail("tests/compile-fail/**/*.rs");
-
-    // `trybuild` will run its tests on `drop`
-    // We want to get a chance to use its output first
-    drop(t);
-
-    overwrite_stderr_files("tests/compile-fail").unwrap();
 }
 
 #[test]
@@ -28,81 +14,3 @@ fn pass() {
     let t = trybuild::TestCases::new();
     t.pass("tests/compile-pass/**/*.rs");
 }
-
-// Compiler messages may change between versions
-// We don't want to have to track these too closely for `bitflags`, but
-// having some message to check makes sure user-facing errors are sensical.
-// 
-// The approach we use is to run the test on all compilers, but only check stderr
-// output on beta (which is the next stable release). We do this by default ignoring
-// any `.stderr` files in the `compile-fail` directory, and copying `.stderr.beta` files
-// when we happen to be running on a beta compiler.
-fn prepare_stderr_files(path: impl AsRef<Path>) -> io::Result<()> {
-    for entry in WalkDir::new(path) {
-        let entry = entry?;
-
-        if entry.path().extension().and_then(OsStr::to_str) == Some("beta") {
-            let renamed = entry.path().with_extension("");
-
-            // Unconditionally remove a corresponding `.stderr` file for a `.stderr.beta`
-            // file if it exists. On `beta` compilers, we'll recreate it. On other compilers,
-            // we don't want to end up checking it anyways.
-            if renamed.exists() {
-                fs::remove_file(&renamed)?;
-            }
-
-            rename_beta_stderr(entry.path(), renamed)?;
-        }
-    }
-
-    Ok(())
-}
-
-// If we want to overwrite the expected compiler output then rename it
-// to use our `.stderr.beta` convention. Otherwise the renamed file won't
-// actually get picked up on the next run
-fn overwrite_stderr_files(path: impl AsRef<Path>) -> io::Result<()> {
-    if env::var("TRYBUILD").ok().filter(|o| o == "overwrite").is_some() {
-        for entry in WalkDir::new(path) {
-            let entry = entry?;
-    
-            // Look for any `.stderr` files and rename them to `.stderr.beta`
-            // If there's an existing `.beta` file then we want to remove it
-            if entry.path().extension().and_then(OsStr::to_str) == Some("stderr") {
-                let renamed = entry.path().with_extension("stderr.beta");
-
-                if renamed.exists() {
-                    remove_beta_stderr(&renamed)?;
-                }
-    
-                rename_beta_stderr(entry.path(), renamed)?;
-            }
-        }
-    }
-
-    Ok(())
-}
-
-#[rustversion::beta]
-fn remove_beta_stderr(path: impl AsRef<Path>) -> io::Result<()> {
-    fs::remove_file(path)?;
-
-    Ok(())
-}
-
-#[rustversion::not(beta)]
-fn remove_beta_stderr(_: impl AsRef<Path>) -> io::Result<()> {
-    Ok(())
-}
-
-#[rustversion::beta]
-fn rename_beta_stderr(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
-    fs::copy(from, to)?;
-
-    Ok(())
-}
-
-#[rustversion::not(beta)]
-fn rename_beta_stderr(_: impl AsRef<Path>, _: impl AsRef<Path>) -> io::Result<()> {
-    Ok(())
-}