Update benchmark results in README with latest data

Enhance benchmark result reporting by checking for expected fixture types and logging failures
2026-04-06 06:39:44 +00:00 · 2026-04-06 06:35:31 +00:00 · 2026-04-05 21:51:44 +00:00 · 2026-04-05 18:09:18 +00:00
2 changed files with 84 additions and 27 deletions
@@ -15,7 +15,7 @@ Only the content between the markers below is rewritten, so the rest of this REA

 <!-- BENCHMARK_RESULTS_START -->

-Last updated: 2026-04-05T17:50:24.432296204Z
+Last updated: 2026-04-06T06:39:34.312411714Z

 ### System info

@@ -39,36 +39,40 @@ Last updated: 2026-04-05T17:50:24.432296204Z

 | Rank | Fixture | Score | Error | Unit |
 |---:|---|---:|---:|---|
-| 1 | `in-memory` | 1089.340 | 31.288 | `ops/us` |
-| 2 | `memory-mapped-file` | 79.877 | 2.664 | `ops/us` |
-| 3 | `datastore4j` | 62.734 | 2.530 | `ops/us` |
-| 4 | `leveldb` | 31.035 | 4.986 | `ops/us` |
-| 5 | `mapdb` | 30.965 | 1.710 | `ops/us` |
-| 6 | `sqlite-jdbc-memory` | 4.250 | 0.107 | `ops/us` |
-| 7 | `lmdb` | 3.350 | 1.844 | `ops/us` |
-| 8 | `kryo` | 2.648 | 0.213 | `ops/us` |
-| 9 | `gson` | 2.314 | 0.125 | `ops/us` |
-| 10 | `sqlite-jdbc` | 2.032 | 0.320 | `ops/us` |
-| 11 | `sqlite-ormlite-memory` | 1.215 | 0.049 | `ops/us` |
-| 12 | `sqlite-ormlite` | 0.931 | 0.037 | `ops/us` |
-| 13 | `duckdb-jdbc` | 0.054 | 0.002 | `ops/us` |
+| 1 | `in-memory` | 1093.921 | 12.329 | `ops/us` |
+| 2 | `memory-mapped-file` | 80.113 | 1.513 | `ops/us` |
+| 3 | `datastore4j` | 59.887 | 2.537 | `ops/us` |
+| 4 | `leveldb` | 35.927 | 0.868 | `ops/us` |
+| 5 | `mapdb` | 31.251 | 1.159 | `ops/us` |
+| 6 | `sqlite-jdbc-memory` | 4.285 | 0.286 | `ops/us` |
+| 7 | `lmdb` | 3.372 | 2.250 | `ops/us` |
+| 8 | `kryo` | 2.722 | 0.171 | `ops/us` |
+| 9 | `gson` | 2.342 | 0.077 | `ops/us` |
+| 10 | `sqlite-jdbc` | 2.061 | 0.052 | `ops/us` |
+| 11 | `sqlite-ormlite-memory` | 1.229 | 0.034 | `ops/us` |
+| 12 | `sqlite-ormlite` | 0.948 | 0.036 | `ops/us` |
+| 13 | `duckdb-jdbc` | 0.054 | 0.005 | `ops/us` |
+| - | `ikv-store` | FAILED | - | `n/a` |
+| - | `chronicle-map` | FAILED | - | `n/a` |

 #### AverageTime

 | Rank | Fixture | Score | Error | Unit |
 |---:|---|---:|---:|---|
 | 1 | `in-memory` | 0.011 | 0.000 | `us/op` |
-| 2 | `memory-mapped-file` | 0.151 | 0.005 | `us/op` |
-| 3 | `datastore4j` | 0.190 | 0.013 | `us/op` |
-| 4 | `leveldb` | 0.330 | 0.017 | `us/op` |
-| 5 | `mapdb` | 0.583 | 0.066 | `us/op` |
-| 6 | `sqlite-jdbc-memory` | 2.885 | 0.233 | `us/op` |
-| 7 | `lmdb` | 3.364 | 1.158 | `us/op` |
-| 8 | `kryo` | 4.503 | 0.211 | `us/op` |
-| 9 | `gson` | 5.175 | 0.574 | `us/op` |
-| 10 | `sqlite-jdbc` | 5.827 | 0.302 | `us/op` |
-| 11 | `sqlite-ormlite-memory` | 9.907 | 0.441 | `us/op` |
-| 12 | `sqlite-ormlite` | 12.929 | 0.427 | `us/op` |
-| 13 | `duckdb-jdbc` | 226.375 | 13.410 | `us/op` |
+| 2 | `memory-mapped-file` | 0.149 | 0.004 | `us/op` |
+| 3 | `datastore4j` | 0.194 | 0.003 | `us/op` |
+| 4 | `leveldb` | 0.357 | 0.015 | `us/op` |
+| 5 | `mapdb` | 0.557 | 0.014 | `us/op` |
+| 6 | `sqlite-jdbc-memory` | 2.820 | 0.056 | `us/op` |
+| 7 | `lmdb` | 3.752 | 1.071 | `us/op` |
+| 8 | `kryo` | 4.420 | 0.336 | `us/op` |
+| 9 | `gson` | 5.168 | 0.582 | `us/op` |
+| 10 | `sqlite-jdbc` | 5.783 | 0.145 | `us/op` |
+| 11 | `sqlite-ormlite-memory` | 9.801 | 0.351 | `us/op` |
+| 12 | `sqlite-ormlite` | 12.611 | 0.307 | `us/op` |
+| 13 | `duckdb-jdbc` | 222.187 | 5.408 | `us/op` |
+| - | `ikv-store` | FAILED | - | `n/a` |
+| - | `chronicle-map` | FAILED | - | `n/a` |

 <!-- BENCHMARK_RESULTS_END -->
@@ -2,6 +2,7 @@ package com.benchmark.runner;

 import com.benchmark.benchmarks.DataAccessBenchmark;
 import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.results.Result;
 import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.runner.options.CommandLineOptions;
@@ -19,11 +20,14 @@ import java.nio.file.Paths;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;

 public class BenchmarkMain {
@@ -68,7 +72,11 @@ public class BenchmarkMain {
                "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
                "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
                "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
-                "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"
+                "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
+                "--add-opens=chronicle.map/net.openhft.chronicle.map=chronicle.core",
+                "--add-opens=chronicle.map/net.openhft.chronicle.map=ALL-UNNAMED",
+                "--add-opens=chronicle.map/net.openhft.chronicle.hash=chronicle.core",
+                "--add-opens=chronicle.map/net.openhft.chronicle.hash=ALL-UNNAMED"
            );

        if (args.length == 0) {
@@ -116,6 +124,24 @@ public class BenchmarkMain {
                    result.getScoreUnit()
                );
            }
+
+            Set<String> presentFixtures = new HashSet<>();
+            for (RunResult runResult : sorted) {
+                presentFixtures.add(runResult.getParams().getParam("fixtureType"));
+            }
+
+            for (String expectedFixture : expectedFixtureTypes()) {
+                if (presentFixtures.contains(expectedFixture)) {
+                    continue;
+                }
+                System.out.printf(
+                    "%-24s %14s %14s %s%n",
+                    expectedFixture,
+                    "FAILED",
+                    "-",
+                    "n/a"
+                );
+            }
        }
    }

@@ -280,12 +306,39 @@ public class BenchmarkMain {
                    .append(result.getScoreUnit())
                    .append("` |\n");
            }
+
+            Set<String> presentFixtures = new HashSet<>();
+            for (RunResult runResult : sorted) {
+                presentFixtures.add(runResult.getParams().getParam("fixtureType"));
+            }
+
+            for (String expectedFixture : expectedFixtureTypes()) {
+                if (presentFixtures.contains(expectedFixture)) {
+                    continue;
+                }
+                sb.append("| - | `")
+                    .append(expectedFixture)
+                    .append("` | FAILED | - | `n/a` |\n");
+            }
            sb.append("\n");
        }

        return sb.toString().trim();
    }

+    private static List<String> expectedFixtureTypes() {
+        try {
+            java.lang.reflect.Field fixtureField = DataAccessBenchmark.class.getDeclaredField("fixtureType");
+            Param param = fixtureField.getAnnotation(Param.class);
+            if (param == null || param.value() == null || param.value().length == 0) {
+                return List.of();
+            }
+            return Arrays.asList(param.value());
+        } catch (NoSuchFieldException e) {
+            return List.of();
+        }
+    }
+
    private static String formatDouble(double value) {
        if (Double.isNaN(value)) {
            return "NaN";
Author	SHA1	Message	Date
Mikhail Yevchenko	280b94c973	Update benchmark results in README with latest data	2026-04-06 06:39:44 +00:00
Mikhail Yevchenko	845bcbfc8d	Enhance benchmark result reporting by checking for expected fixture types and logging failures	2026-04-06 06:35:31 +00:00
Mikhail Yevchenko	d3d29fd60f	Update benchmark results in README with latest data	2026-04-05 21:51:44 +00:00
Mikhail Yevchenko	fb013bbd11	Add ChronicleMap JVM --add-opens flags	2026-04-05 18:09:18 +00:00