From 2a4c51b77dcc2036468f40f7b8855229daaba1b0 Mon Sep 17 00:00:00 2001
From: Pat Hickey <pat@moreproductive.org>
Date: Fri, 4 Jun 2021 12:24:44 -0700
Subject: [PATCH] switch eager vs lazy instantiation to a criterion bench

---
 Cargo.toml                                  |  4 ++
 {tests/all => benches}/thread_eager_init.rs | 74 +++++++++++++--------
 tests/all/main.rs                           |  1 -
 3 files changed, 51 insertions(+), 28 deletions(-)
 rename {tests/all => benches}/thread_eager_init.rs (53%)

diff --git a/Cargo.toml b/Cargo.toml
index 9b1ce39b16..712b57bb09 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -120,3 +120,7 @@ debug = false # FIXME(#1813)
 [[bench]]
 name = "instantiation"
 harness = false
+
+[[bench]]
+name = "thread_eager_init"
+harness = false
diff --git a/tests/all/thread_eager_init.rs b/benches/thread_eager_init.rs
similarity index 53%
rename from tests/all/thread_eager_init.rs
rename to benches/thread_eager_init.rs
index 63cee33e6d..9d35869cea 100644
--- a/tests/all/thread_eager_init.rs
+++ b/benches/thread_eager_init.rs
@@ -1,11 +1,47 @@
-use anyhow::Result;
+use criterion::{criterion_group, criterion_main, Criterion};
 use std::thread;
 use std::time::{Duration, Instant};
 use wasmtime::*;
 
-#[test]
-fn measure_execution_time() -> Result<()> {
-    let iterations = 1000;
+fn measure_execution_time(c: &mut Criterion) {
+    c.bench_function("lazy initialization at call", move |b| {
+        let (engine, module) = test_engine();
+        b.iter_custom(move |iters| {
+            (0..iters)
+                .into_iter()
+                .map(|_| lazy_thread_instantiate(engine.clone(), module.clone()))
+                .sum()
+        })
+    });
+
+    c.bench_function("eager initialization", move |b| {
+        let (engine, module) = test_engine();
+        b.iter_custom(move |iters| {
+            (0..iters)
+                .into_iter()
+                .map(|_| {
+                    let (init, _call) = eager_thread_instantiate(engine.clone(), module.clone());
+                    init
+                })
+                .sum()
+        })
+    });
+    c.bench_function("call after eager initialization", move |b| {
+        let (engine, module) = test_engine();
+        b.iter_custom(move |iters| {
+            (0..iters)
+                .into_iter()
+                .map(|_| {
+                    let (_init, call) = eager_thread_instantiate(engine.clone(), module.clone());
+                    call
+                })
+                .sum()
+        })
+    });
+}
+
+fn test_engine() -> (Engine, Module) {
+    let pool_count = 1000;
 
     let mut config = Config::new();
     config.allocation_strategy(InstanceAllocationStrategy::Pooling {
@@ -15,33 +51,14 @@ fn measure_execution_time() -> Result<()> {
             ..Default::default()
         },
         instance_limits: InstanceLimits {
-            count: iterations * 2,
+            count: pool_count,
             memory_reservation_size: 1,
         },
     });
 
-    let engine = Engine::new(&config)?;
-    let module = Module::new(&engine, r#"(module (memory 1) (func (export "f")))"#)?;
-
-    let lazy_call_time: Duration = (0..iterations)
-        .into_iter()
-        .map(|_| lazy_thread_instantiate(engine.clone(), module.clone()))
-        .sum();
-
-    let (eager_init_total, eager_call_total): (Duration, Duration) = (0..iterations)
-        .into_iter()
-        .map(|_| eager_thread_instantiate(engine.clone(), module.clone()))
-        .fold(
-            (Duration::default(), Duration::default()),
-            |(s1, s2), (d1, d2)| (s1 + d1, s2 + d2),
-        );
-
-    println!(
-        "lazy call: {:?}, eager init: {:?}, eager call: {:?}",
-        lazy_call_time, eager_init_total, eager_call_total
-    );
-
-    Ok(())
+    let engine = Engine::new(&config).unwrap();
+    let module = Module::new(&engine, r#"(module (memory 1) (func (export "f")))"#).unwrap();
+    (engine, module)
 }
 
 fn lazy_thread_instantiate(engine: Engine, module: Module) -> Duration {
@@ -77,3 +94,6 @@ fn eager_thread_instantiate(engine: Engine, module: Module) -> (Duration, Durati
     .join()
     .expect("thread joins")
 }
+
+criterion_group!(benches, measure_execution_time);
+criterion_main!(benches);
diff --git a/tests/all/main.rs b/tests/all/main.rs
index a1f3314d64..37bb68d7a6 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -24,7 +24,6 @@ mod pooling_allocator;
 mod stack_overflow;
 mod store;
 mod table;
-mod thread_eager_init;
 mod traps;
 mod wast;