Add nextest compatibility (#602)

.github/workflows/ci.yaml

@@ -8,6 +8,8 @@ on:
       - version-0.4
 
 name: tests
+env:
+  CARGO_TERM_COLOR: always
 
 jobs:
   ci:
@@ -85,3 +87,27 @@ jobs:
           export PATH=$HOME/.wasmtime/bin/:$PATH
           cargo install cargo-wasi
           cargo wasi bench --no-default-features -- --test
+
+  nextest-compat:
+    name: Check compatibility with nextest
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v1
+
+      - uses: taiki-e/install-action@nextest
+
+      - run: ci/nextest-compat.sh

ci/nextest-compat.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -ex -o pipefail
+
+CARGO=${CARGO:-cargo}
+
+cd "$(git rev-parse --show-toplevel)"
+
+echo "Checking benches/bench_main..."
+
+$CARGO nextest list --benches
+$CARGO nextest run --benches

src/benchmark_group.rs

@@ -328,9 +328,9 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
                     );
                 }
             }
-            Mode::List => {
+            Mode::List(_) => {
                 if do_run {
-                    println!("{}: bench", id);
+                    println!("{}: benchmark", id);
                 }
             }
             Mode::Test => {
@@ -392,7 +392,7 @@ impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> {
                 self.criterion.measurement.formatter(),
             );
         }
-        if self.any_matched {
+        if self.any_matched && !self.criterion.mode.is_terse() {
             self.criterion.report.group_separator();
         }
     }

src/lib.rs

@@ -108,8 +108,8 @@ static DEFAULT_PLOTTING_BACKEND: Lazy<PlottingBackend> = Lazy::new(|| match &*GN
     #[cfg(feature = "plotters")]
     Err(e) => {
         match e {
-            VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"),
-            e => println!(
+            VersionError::Exec(_) => eprintln!("Gnuplot not found, using plotters backend"),
+            e => eprintln!(
                 "Gnuplot not found or not usable, using plotters backend\n{}",
                 e
             ),
@@ -303,7 +303,7 @@ pub(crate) enum Mode {
     /// Run benchmarks normally.
     Benchmark,
     /// List all benchmarks but do not run them.
-    List,
+    List(ListFormat),
     /// Run benchmarks once to verify that they work, but otherwise do not measure them.
     Test,
     /// Iterate benchmarks for a given length of time but do not analyze or report on them.
@@ -313,6 +313,39 @@ impl Mode {
     pub fn is_benchmark(&self) -> bool {
         matches!(self, Mode::Benchmark)
     }
+
+    pub fn is_terse(&self) -> bool {
+        matches!(self, Mode::List(ListFormat::Terse))
+    }
+}
+
+#[derive(Debug, Clone)]
+/// Enum representing the list format.
+pub(crate) enum ListFormat {
+    /// The regular, default format.
+    Pretty,
+    /// The terse format, where nothing other than the name of the test and ": benchmark" at the end
+    /// is printed out.
+    Terse,
+}
+
+impl Default for ListFormat {
+    fn default() -> Self {
+        Self::Pretty
+    }
+}
+
+/// Benchmark filtering support.
+#[derive(Clone, Debug)]
+pub enum BenchmarkFilter {
+    /// Run all benchmarks.
+    AcceptAll,
+    /// Run benchmarks matching this regex.
+    Regex(Regex),
+    /// Run the benchmark matching this string exactly.
+    Exact(String),
+    /// Do not run any benchmarks.
+    RejectAll,
 }
 
 /// The benchmark manager
@@ -331,7 +364,7 @@ impl Mode {
 /// benchmark.
 pub struct Criterion<M: Measurement = WallTime> {
     config: BenchmarkConfig,
-    filter: Option<Regex>,
+    filter: BenchmarkFilter,
     report: Reports,
     output_directory: PathBuf,
     baseline_directory: String,
@@ -399,7 +432,7 @@ impl Default for Criterion {
                 sampling_mode: SamplingMode::Auto,
                 quick_mode: false,
             },
-            filter: None,
+            filter: BenchmarkFilter::AcceptAll,
             report: reports,
             baseline_directory: "base".to_owned(),
             baseline: Baseline::Save,
@@ -660,6 +693,8 @@ impl<M: Measurement> Criterion<M> {
     #[must_use]
     /// Filters the benchmarks. Only benchmarks with names that contain the
     /// given string will be executed.
+    ///
+    /// This overwrites [`Self::with_benchmark_filter`].
     pub fn with_filter<S: Into<String>>(mut self, filter: S) -> Criterion<M> {
         let filter_text = filter.into();
         let filter = Regex::new(&filter_text).unwrap_or_else(|err| {
@@ -668,7 +703,16 @@ impl<M: Measurement> Criterion<M> {
                 filter_text, err
             )
         });
-        self.filter = Some(filter);
+        self.filter = BenchmarkFilter::Regex(filter);
+
+        self
+    }
+
+    /// Only run benchmarks specified by the given filter.
+    ///
+    /// This overwrites [`Self::with_filter`].
+    pub fn with_benchmark_filter(mut self, filter: BenchmarkFilter) -> Criterion<M> {
+        self.filter = filter;
 
         self
     }
@@ -771,6 +815,19 @@ impl<M: Measurement> Criterion<M> {
                 .long("list")
                 .help("List all benchmarks")
                 .conflicts_with_all(&["test", "profile-time"]))
+            .arg(Arg::new("format")
+                .long("format")
+                .possible_values(["pretty", "terse"])
+                .default_value("pretty")
+                // Note that libtest's --format also works during test execution, but criterion
+                // doesn't support that at the moment.
+                .help("Output formatting"))
+            .arg(Arg::new("ignored")
+                .long("ignored")
+                .help("List or run ignored benchmarks (currently means skip all benchmarks)"))
+            .arg(Arg::new("exact")
+                .long("exact")
+                .help("Run benchmarks that exactly match the provided filter"))
             .arg(Arg::new("profile-time")
                 .long("profile-time")
                 .takes_value(true)
@@ -901,10 +958,21 @@ https://bheisler.github.io/criterion.rs/book/faq.html
             (false, _) => true,     // cargo test --benches should run tests
         };
 
-        self.mode = if test_mode {
+        self.mode = if matches.is_present("list") {
+            let list_format = match matches
+                .value_of("format")
+                .expect("a default value was provided for this")
+            {
+                "pretty" => ListFormat::Pretty,
+                "terse" => ListFormat::Terse,
+                other => unreachable!(
+                    "unrecognized value for --format that isn't part of possible-values: {}",
+                    other
+                ),
+            };
+            Mode::List(list_format)
+        } else if test_mode {
             Mode::Test
-        } else if matches.is_present("list") {
-            Mode::List
         } else if matches.is_present("profile-time") {
             let num_seconds = matches.value_of_t_or_exit("profile-time");
 
@@ -923,9 +991,25 @@ https://bheisler.github.io/criterion.rs/book/faq.html
             self.connection = None;
         }
 
-        if let Some(filter) = matches.value_of("FILTER") {
-            self = self.with_filter(filter);
-        }
+        let filter = if matches.is_present("ignored") {
+            // --ignored overwrites any name-based filters passed in.
+            BenchmarkFilter::RejectAll
+        } else if let Some(filter) = matches.value_of("FILTER") {
+            if matches.is_present("exact") {
+                BenchmarkFilter::Exact(filter.to_owned())
+            } else {
+                let regex = Regex::new(filter).unwrap_or_else(|err| {
+                    panic!(
+                        "Unable to parse '{}' as a regular expression: {}",
+                        filter, err
+                    )
+                });
+                BenchmarkFilter::Regex(regex)
+            }
+        } else {
+            BenchmarkFilter::AcceptAll
+        };
+        self = self.with_benchmark_filter(filter);
 
         match matches.value_of("plotting-backend") {
             // Use plotting_backend() here to re-use the panic behavior if Gnuplot is not available.
@@ -1061,8 +1145,10 @@ https://bheisler.github.io/criterion.rs/book/faq.html
 
     fn filter_matches(&self, id: &str) -> bool {
         match &self.filter {
-            Some(regex) => regex.is_match(id),
-            None => true,
+            BenchmarkFilter::AcceptAll => true,
+            BenchmarkFilter::Regex(regex) => regex.is_match(id),
+            BenchmarkFilter::Exact(exact) => id == exact,
+            BenchmarkFilter::RejectAll => false,
         }
     }