Update benchmark results in README with latest data

README.md

@@ -15,7 +15,7 @@ Only the content between the markers below is rewritten, so the rest of this REA
 
 <!-- BENCHMARK_RESULTS_START -->
 
-Last updated: 2026-04-05T17:19:53.068450463Z
+Last updated: 2026-04-06T06:39:34.312411714Z
 
 ### System info
 
@@ -31,7 +31,7 @@ Last updated: 2026-04-05T17:19:53.068450463Z
 
 - Benchmark: `com.benchmark.benchmarks.DataAccessBenchmark.readSingleUser`
 - Thread count: `12`
-- Fixtures in this snapshot: `30`
+- Fixtures in this snapshot: `26`
 
 ### Results
 
@@ -39,40 +39,40 @@ Last updated: 2026-04-05T17:19:53.068450463Z
 
 | Rank | Fixture | Score | Error | Unit |
 |---:|---|---:|---:|---|
-| 1 | `ikv-store` | 1085.094 | 32.267 | `ops/us` |
+| 1 | `in-memory` | 1093.921 | 12.329 | `ops/us` |
-| 2 | `in-memory` | 1076.857 | 17.298 | `ops/us` |
+| 2 | `memory-mapped-file` | 80.113 | 1.513 | `ops/us` |
-| 3 | `chronicle-map` | 85.276 | 2.529 | `ops/us` |
+| 3 | `datastore4j` | 59.887 | 2.537 | `ops/us` |
-| 4 | `memory-mapped-file` | 79.549 | 1.791 | `ops/us` |
+| 4 | `leveldb` | 35.927 | 0.868 | `ops/us` |
-| 5 | `datastore4j` | 61.163 | 2.313 | `ops/us` |
+| 5 | `mapdb` | 31.251 | 1.159 | `ops/us` |
-| 6 | `leveldb` | 31.796 | 5.871 | `ops/us` |
+| 6 | `sqlite-jdbc-memory` | 4.285 | 0.286 | `ops/us` |
-| 7 | `mapdb` | 28.615 | 6.765 | `ops/us` |
+| 7 | `lmdb` | 3.372 | 2.250 | `ops/us` |
-| 8 | `sqlite-jdbc-memory` | 4.153 | 0.249 | `ops/us` |
+| 8 | `kryo` | 2.722 | 0.171 | `ops/us` |
-| 9 | `lmdb` | 3.115 | 0.864 | `ops/us` |
+| 9 | `gson` | 2.342 | 0.077 | `ops/us` |
-| 10 | `kryo` | 2.549 | 0.150 | `ops/us` |
+| 10 | `sqlite-jdbc` | 2.061 | 0.052 | `ops/us` |
-| 11 | `gson` | 2.154 | 0.183 | `ops/us` |
+| 11 | `sqlite-ormlite-memory` | 1.229 | 0.034 | `ops/us` |
-| 12 | `sqlite-jdbc` | 1.970 | 0.162 | `ops/us` |
+| 12 | `sqlite-ormlite` | 0.948 | 0.036 | `ops/us` |
-| 13 | `sqlite-ormlite-memory` | 1.071 | 0.255 | `ops/us` |
+| 13 | `duckdb-jdbc` | 0.054 | 0.005 | `ops/us` |
-| 14 | `sqlite-ormlite` | 0.886 | 0.039 | `ops/us` |
+| - | `ikv-store` | FAILED | - | `n/a` |
-| 15 | `duckdb-jdbc` | 0.050 | 0.004 | `ops/us` |
+| - | `chronicle-map` | FAILED | - | `n/a` |
 
 #### AverageTime
 
 | Rank | Fixture | Score | Error | Unit |
 |---:|---|---:|---:|---|
-| 1 | `ikv-store` | 0.011 | 0.000 | `us/op` |
+| 1 | `in-memory` | 0.011 | 0.000 | `us/op` |
-| 2 | `in-memory` | 0.011 | 0.000 | `us/op` |
+| 2 | `memory-mapped-file` | 0.149 | 0.004 | `us/op` |
-| 3 | `chronicle-map` | 0.142 | 0.005 | `us/op` |
+| 3 | `datastore4j` | 0.194 | 0.003 | `us/op` |
-| 4 | `memory-mapped-file` | 0.154 | 0.004 | `us/op` |
+| 4 | `leveldb` | 0.357 | 0.015 | `us/op` |
-| 5 | `datastore4j` | 0.204 | 0.010 | `us/op` |
+| 5 | `mapdb` | 0.557 | 0.014 | `us/op` |
-| 6 | `leveldb` | 0.387 | 0.046 | `us/op` |
+| 6 | `sqlite-jdbc-memory` | 2.820 | 0.056 | `us/op` |
-| 7 | `mapdb` | 0.403 | 0.031 | `us/op` |
+| 7 | `lmdb` | 3.752 | 1.071 | `us/op` |
-| 8 | `sqlite-jdbc-memory` | 3.005 | 0.504 | `us/op` |
+| 8 | `kryo` | 4.420 | 0.336 | `us/op` |
-| 9 | `lmdb` | 3.665 | 0.355 | `us/op` |
+| 9 | `gson` | 5.168 | 0.582 | `us/op` |
-| 10 | `kryo` | 4.820 | 0.338 | `us/op` |
+| 10 | `sqlite-jdbc` | 5.783 | 0.145 | `us/op` |
-| 11 | `gson` | 5.416 | 0.368 | `us/op` |
+| 11 | `sqlite-ormlite-memory` | 9.801 | 0.351 | `us/op` |
-| 12 | `sqlite-jdbc` | 6.056 | 0.441 | `us/op` |
+| 12 | `sqlite-ormlite` | 12.611 | 0.307 | `us/op` |
-| 13 | `sqlite-ormlite-memory` | 10.431 | 0.868 | `us/op` |
+| 13 | `duckdb-jdbc` | 222.187 | 5.408 | `us/op` |
-| 14 | `sqlite-ormlite` | 13.394 | 0.376 | `us/op` |
+| - | `ikv-store` | FAILED | - | `n/a` |
-| 15 | `duckdb-jdbc` | 235.873 | 22.582 | `us/op` |
+| - | `chronicle-map` | FAILED | - | `n/a` |
 
 <!-- BENCHMARK_RESULTS_END -->

src/main/java/com/benchmark/fixtures/ChronicleMapFixture.java

@@ -8,14 +8,11 @@ import net.openhft.chronicle.map.ChronicleMap;
 import net.openhft.chronicle.map.ChronicleMapBuilder;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public class ChronicleMapFixture implements DataFixtures {
 
     private ChronicleMap<Long, byte[]> usersById;
-    private Map<Long, byte[]> fallbackUsersById;
     private Kryo kryo;
 
     @Override
@@ -24,11 +21,9 @@ public class ChronicleMapFixture implements DataFixtures {
         kryo.setRegistrationRequired(false);
         kryo.register(User.class);
 
-        Map<Long, byte[]> encodedUsers = new HashMap<>(users.size());
         int maxValueSize = 1;
         for (User user : users) {
             byte[] encoded = serialize(user);
-            encodedUsers.put(user.getId(), encoded);
             maxValueSize = Math.max(maxValueSize, encoded.length);
         }
 
@@ -40,19 +35,22 @@ public class ChronicleMapFixture implements DataFixtures {
                 .averageValue(new byte[maxValueSize])
                 .create();
 
-            for (Map.Entry<Long, byte[]> entry : encodedUsers.entrySet()) {
+            for (User user : users) {
-                usersById.put(entry.getKey(), entry.getValue());
+                usersById.put(user.getId(), serialize(user));
             }
-            fallbackUsersById = null;
+        } catch (Throwable t) {
-        } catch (Throwable ignored) {
             usersById = null;
-            fallbackUsersById = encodedUsers;
+            throw new IllegalStateException(
+                "ChronicleMap initialization failed. Cause: "
+                    + t.getClass().getSimpleName() + " - " + t.getMessage(),
+                t
+            );
         }
     }
 
     @Override
     public User findById(long id) {
-        byte[] encoded = usersById != null ? usersById.get(id) : fallbackUsersById.get(id);
+        byte[] encoded = usersById.get(id);
         return encoded == null ? null : deserialize(encoded);
     }
 
@@ -61,7 +59,6 @@ public class ChronicleMapFixture implements DataFixtures {
         if (usersById != null) {
             usersById.close();
         }
-        fallbackUsersById = null;
     }
 
     private byte[] serialize(User user) {

src/main/java/com/benchmark/fixtures/IkvStoreFixture.java

@@ -4,17 +4,11 @@ import com.benchmark.model.User;
 
 import java.lang.reflect.Method;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 public class IkvStoreFixture implements DataFixtures {
 
-    private static final AtomicBoolean FALLBACK_WARNING_PRINTED = new AtomicBoolean(false);
-
-    private final InMemoryFixture fallback = new InMemoryFixture();
-
     private Object reader;
     private Object writer;
-    private boolean useFallback;
 
     @Override
     public void setup(List<User> users) throws Exception {
@@ -24,10 +18,10 @@ public class IkvStoreFixture implements DataFixtures {
         String mountDirectory = getenv("IKV_MOUNT_DIRECTORY");
 
         if (accountId == null || accountPassKey == null || storeName == null || mountDirectory == null) {
-            useFallback = true;
+            throw new IllegalStateException(
-            fallback.setup(users);
+                "IKV fixture requires IKV_ACCOUNT_ID, IKV_ACCOUNT_PASSKEY, IKV_STORE_NAME, and IKV_MOUNT_DIRECTORY. "
-            printFallbackWarning("IKV fixture: required env vars missing (IKV_ACCOUNT_ID, IKV_ACCOUNT_PASSKEY, IKV_STORE_NAME, IKV_MOUNT_DIRECTORY). Using in-memory fallback.");
+                    + "Results are invalid without a real IKV backend."
-            return;
+            );
         }
 
         try {
@@ -80,25 +74,21 @@ public class IkvStoreFixture implements DataFixtures {
 
             // IKV is eventually consistent for reads. Small delay before measurement starts.
             Thread.sleep(200);
-            useFallback = false;
         } catch (Throwable t) {
-            useFallback = true;
-            fallback.setup(users);
             closeQuietly(reader, "shutdownReader");
             closeQuietly(writer, "shutdownWriter");
             reader = null;
             writer = null;
-            printFallbackWarning("IKV fixture: initialization failed, using in-memory fallback. Cause: "
+            throw new IllegalStateException(
-                + t.getClass().getSimpleName() + " - " + t.getMessage());
+                "IKV fixture initialization failed. Cause: "
+                    + t.getClass().getSimpleName() + " - " + t.getMessage(),
+                t
+            );
         }
     }
 
     @Override
     public User findById(long id) throws Exception {
-        if (useFallback) {
-            return fallback.findById(id);
-        }
-
         String key = Long.toString(id);
         String name = (String) reader.getClass().getMethod("getStringValue", String.class, String.class)
             .invoke(reader, key, "name");
@@ -117,11 +107,6 @@ public class IkvStoreFixture implements DataFixtures {
 
     @Override
     public void teardown() throws Exception {
-        if (useFallback) {
-            fallback.teardown();
-            return;
-        }
-
         closeQuietly(reader, "shutdownReader");
         closeQuietly(writer, "shutdownWriter");
         reader = null;
@@ -143,10 +128,4 @@ public class IkvStoreFixture implements DataFixtures {
             // Ignore cleanup issues for benchmark fixture shutdown.
         }
     }
-
-    private static void printFallbackWarning(String message) {
-        if (FALLBACK_WARNING_PRINTED.compareAndSet(false, true)) {
-            System.out.println(message);
-        }
-    }
 }

src/main/java/com/benchmark/runner/BenchmarkMain.java

@@ -2,6 +2,7 @@ package com.benchmark.runner;
 
 import com.benchmark.benchmarks.DataAccessBenchmark;
 import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.runner.options.CommandLineOptions;
@@ -19,11 +20,14 @@ import java.nio.file.Paths;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 public class BenchmarkMain {
@@ -68,7 +72,11 @@ public class BenchmarkMain {
                 "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
                 "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
                 "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
-                "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"
+                "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
+                "--add-opens=chronicle.map/net.openhft.chronicle.map=chronicle.core",
+                "--add-opens=chronicle.map/net.openhft.chronicle.map=ALL-UNNAMED",
+                "--add-opens=chronicle.map/net.openhft.chronicle.hash=chronicle.core",
+                "--add-opens=chronicle.map/net.openhft.chronicle.hash=ALL-UNNAMED"
             );
 
         if (args.length == 0) {
@@ -116,6 +124,24 @@ public class BenchmarkMain {
                     result.getScoreUnit()
                 );
             }
+
+            Set<String> presentFixtures = new HashSet<>();
+            for (RunResult runResult : sorted) {
+                presentFixtures.add(runResult.getParams().getParam("fixtureType"));
+            }
+
+            for (String expectedFixture : expectedFixtureTypes()) {
+                if (presentFixtures.contains(expectedFixture)) {
+                    continue;
+                }
+                System.out.printf(
+                    "%-24s %14s %14s %s%n",
+                    expectedFixture,
+                    "FAILED",
+                    "-",
+                    "n/a"
+                );
+            }
         }
     }
 
@@ -280,12 +306,39 @@ public class BenchmarkMain {
                     .append(result.getScoreUnit())
                     .append("` |\n");
             }
+
+            Set<String> presentFixtures = new HashSet<>();
+            for (RunResult runResult : sorted) {
+                presentFixtures.add(runResult.getParams().getParam("fixtureType"));
+            }
+
+            for (String expectedFixture : expectedFixtureTypes()) {
+                if (presentFixtures.contains(expectedFixture)) {
+                    continue;
+                }
+                sb.append("| - | `")
+                    .append(expectedFixture)
+                    .append("` | FAILED | - | `n/a` |\n");
+            }
             sb.append("\n");
         }
 
         return sb.toString().trim();
     }
 
+    private static List<String> expectedFixtureTypes() {
+        try {
+            java.lang.reflect.Field fixtureField = DataAccessBenchmark.class.getDeclaredField("fixtureType");
+            Param param = fixtureField.getAnnotation(Param.class);
+            if (param == null || param.value() == null || param.value().length == 0) {
+                return List.of();
+            }
+            return Arrays.asList(param.value());
+        } catch (NoSuchFieldException e) {
+            return List.of();
+        }
+    }
+
     private static String formatDouble(double value) {
         if (Double.isNaN(value)) {
             return "NaN";