Mikhail Yevchenko
463 lines
18 KiB
Java
463 lines
18 KiB
Java
package com.benchmark.runner;
|
|
|
|
import com.benchmark.benchmarks.DataAccessBenchmark;
|
|
import org.openjdk.jmh.annotations.Mode;
|
|
import org.openjdk.jmh.annotations.Param;
|
|
import org.openjdk.jmh.results.Result;
|
|
import org.openjdk.jmh.results.RunResult;
|
|
import org.openjdk.jmh.runner.options.CommandLineOptions;
|
|
import org.openjdk.jmh.runner.Runner;
|
|
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
|
|
import org.openjdk.jmh.runner.options.Options;
|
|
import org.openjdk.jmh.runner.options.OptionsBuilder;
|
|
|
|
import java.lang.management.ManagementFactory;
|
|
import java.io.IOException;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.nio.file.Paths;
|
|
import java.time.ZonedDateTime;
|
|
import java.time.format.DateTimeFormatter;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Collection;
|
|
import java.util.HashSet;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
public class BenchmarkMain {
|
|
|
|
private static final String README_FILE = "README.md";
|
|
private static final String RESULTS_SECTION_START = "<!-- BENCHMARK_RESULTS_START -->";
|
|
private static final String RESULTS_SECTION_END = "<!-- BENCHMARK_RESULTS_END -->";
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
if (isJmhListingOrHelpCommand(args)) {
|
|
org.openjdk.jmh.Main.main(args);
|
|
return;
|
|
}
|
|
|
|
CommandLineOptions commandLineOptions = new CommandLineOptions(args);
|
|
|
|
ChainedOptionsBuilder builder = new OptionsBuilder()
|
|
.parent(commandLineOptions)
|
|
.jvmArgsPrepend(
|
|
"--enable-native-access=ALL-UNNAMED",
|
|
"--sun-misc-unsafe-memory-access=allow",
|
|
"--add-opens=java.base/java.lang=ALL-UNNAMED",
|
|
"--add-opens=java.base/java.nio=ALL-UNNAMED",
|
|
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
|
|
"--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
|
|
"--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
|
|
"--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
|
|
"--add-opens=chronicle.map/net.openhft.chronicle.map=chronicle.core",
|
|
"--add-opens=chronicle.map/net.openhft.chronicle.map=ALL-UNNAMED",
|
|
"--add-opens=chronicle.map/net.openhft.chronicle.hash=chronicle.core",
|
|
"--add-opens=chronicle.map/net.openhft.chronicle.hash=ALL-UNNAMED"
|
|
);
|
|
|
|
if (args.length == 0) {
|
|
builder.include(DataAccessBenchmark.class.getSimpleName());
|
|
}
|
|
|
|
Options opt = builder.build();
|
|
Collection<RunResult> runResults = new Runner(opt).run();
|
|
printSortedSummary(runResults);
|
|
updateReadmeWithLatestResults(runResults);
|
|
}
|
|
|
|
private static void printSortedSummary(Collection<RunResult> runResults) {
|
|
if (runResults == null || runResults.isEmpty()) {
|
|
return;
|
|
}
|
|
|
|
Map<Mode, List<RunResult>> byMode = new LinkedHashMap<>();
|
|
for (RunResult runResult : runResults) {
|
|
byMode.computeIfAbsent(runResult.getParams().getMode(), m -> new ArrayList<>()).add(runResult);
|
|
}
|
|
|
|
System.out.println("\n=== Sorted summary by performance ===");
|
|
for (Map.Entry<Mode, List<RunResult>> entry : byMode.entrySet()) {
|
|
Mode mode = entry.getKey();
|
|
List<RunResult> sorted = new ArrayList<>(entry.getValue());
|
|
boolean lowerIsBetter = mode == Mode.AverageTime || mode == Mode.SampleTime || mode == Mode.SingleShotTime;
|
|
|
|
sorted.sort((a, b) -> {
|
|
double scoreA = a.getPrimaryResult().getScore();
|
|
double scoreB = b.getPrimaryResult().getScore();
|
|
return lowerIsBetter ? Double.compare(scoreA, scoreB) : Double.compare(scoreB, scoreA);
|
|
});
|
|
|
|
System.out.printf("%n[%s]%n", mode);
|
|
System.out.printf("%-24s %14s %14s %s%n", "fixtureType", "score", "error", "unit");
|
|
for (RunResult runResult : sorted) {
|
|
Result<?> result = runResult.getPrimaryResult();
|
|
String fixtureType = runResult.getParams().getParam("fixtureType");
|
|
System.out.printf(
|
|
"%-24s %14.3f %14.3f %s%n",
|
|
fixtureType,
|
|
result.getScore(),
|
|
result.getScoreError(),
|
|
result.getScoreUnit()
|
|
);
|
|
}
|
|
|
|
Set<String> presentFixtures = new HashSet<>();
|
|
for (RunResult runResult : sorted) {
|
|
presentFixtures.add(runResult.getParams().getParam("fixtureType"));
|
|
}
|
|
|
|
for (String expectedFixture : expectedFixtureTypes()) {
|
|
if (presentFixtures.contains(expectedFixture)) {
|
|
continue;
|
|
}
|
|
System.out.printf(
|
|
"%-24s %14s %14s %s%n",
|
|
expectedFixture,
|
|
"FAILED",
|
|
"-",
|
|
"n/a"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static boolean isJmhListingOrHelpCommand(String[] args) {
|
|
for (String arg : args) {
|
|
if ("-l".equals(arg) || "-lp".equals(arg) || "-lprof".equals(arg)
|
|
|| "-h".equals(arg) || "--help".equals(arg)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private static void updateReadmeWithLatestResults(Collection<RunResult> runResults) {
|
|
if (runResults == null || runResults.isEmpty()) {
|
|
return;
|
|
}
|
|
|
|
Path readmePath = Paths.get(README_FILE);
|
|
if (!Files.exists(readmePath)) {
|
|
return;
|
|
}
|
|
|
|
String sectionContent = buildReadmeResultsSection(runResults);
|
|
|
|
try {
|
|
String readme = Files.readString(readmePath, StandardCharsets.UTF_8);
|
|
int start = readme.indexOf(RESULTS_SECTION_START);
|
|
int end = readme.indexOf(RESULTS_SECTION_END);
|
|
|
|
String updated;
|
|
if (start >= 0 && end > start) {
|
|
int contentStart = start + RESULTS_SECTION_START.length();
|
|
updated = readme.substring(0, contentStart)
|
|
+ "\n\n"
|
|
+ sectionContent
|
|
+ "\n\n"
|
|
+ readme.substring(end);
|
|
} else {
|
|
updated = readme
|
|
+ "\n\n## Benchmark Results (auto-updated)\n\n"
|
|
+ "This section is managed by the benchmark runner."
|
|
+ " Running `java -jar target/benchmarks.jar` refreshes the snapshot below.\n\n"
|
|
+ RESULTS_SECTION_START
|
|
+ "\n\n"
|
|
+ sectionContent
|
|
+ "\n\n"
|
|
+ RESULTS_SECTION_END
|
|
+ "\n";
|
|
}
|
|
|
|
Files.writeString(readmePath, updated, StandardCharsets.UTF_8);
|
|
System.out.println("\nUpdated README benchmark snapshot.");
|
|
} catch (IOException e) {
|
|
System.err.println("Failed to update README benchmark snapshot: " + e.getMessage());
|
|
}
|
|
}
|
|
|
|
private static String buildReadmeResultsSection(Collection<RunResult> runResults) {
|
|
StringBuilder sb = new StringBuilder();
|
|
ZonedDateTime now = ZonedDateTime.now();
|
|
|
|
sb.append("Last updated: ")
|
|
.append(now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME))
|
|
.append("\n\n");
|
|
|
|
sb.append("### System info\n\n");
|
|
sb.append("- OS: `")
|
|
.append(System.getProperty("os.name"))
|
|
.append(" ")
|
|
.append(System.getProperty("os.version"))
|
|
.append("`")
|
|
.append(" (")
|
|
.append(System.getProperty("os.arch"))
|
|
.append(")\n");
|
|
sb.append("- Java: `")
|
|
.append(System.getProperty("java.version"))
|
|
.append("` (vendor: `")
|
|
.append(System.getProperty("java.vendor"))
|
|
.append("`)\n");
|
|
sb.append("- JVM: `")
|
|
.append(System.getProperty("java.vm.name"))
|
|
.append("` `")
|
|
.append(System.getProperty("java.vm.version"))
|
|
.append("`\n");
|
|
|
|
String cpuModel = detectCpuModel();
|
|
if (cpuModel != null && !cpuModel.isBlank()) {
|
|
sb.append("- CPU model: `")
|
|
.append(cpuModel)
|
|
.append("`\n");
|
|
}
|
|
|
|
String ramModel = detectRamModel();
|
|
if (ramModel != null && !ramModel.isBlank()) {
|
|
sb.append("- RAM model: `")
|
|
.append(ramModel)
|
|
.append("`\n");
|
|
}
|
|
|
|
long totalPhysicalMemoryBytes = detectTotalPhysicalMemoryBytes();
|
|
if (totalPhysicalMemoryBytes > 0) {
|
|
sb.append("- Total physical memory: `")
|
|
.append(totalPhysicalMemoryBytes / (1024 * 1024))
|
|
.append(" MB`\n");
|
|
}
|
|
|
|
sb.append("- Available processors: `")
|
|
.append(Runtime.getRuntime().availableProcessors())
|
|
.append("`\n");
|
|
sb.append("- Max heap: `")
|
|
.append(Runtime.getRuntime().maxMemory() / (1024 * 1024))
|
|
.append(" MB`\n\n");
|
|
|
|
RunResult first = runResults.iterator().next();
|
|
sb.append("### Run context\n\n");
|
|
sb.append("- Benchmark: `")
|
|
.append(first.getParams().getBenchmark())
|
|
.append("`\n");
|
|
sb.append("- Thread count: `")
|
|
.append(first.getParams().getThreads())
|
|
.append("`\n");
|
|
sb.append("- Fixtures in this snapshot: `")
|
|
.append(runResults.size())
|
|
.append("`\n\n");
|
|
|
|
Map<Mode, List<RunResult>> byMode = new LinkedHashMap<>();
|
|
for (RunResult runResult : runResults) {
|
|
byMode.computeIfAbsent(runResult.getParams().getMode(), m -> new ArrayList<>()).add(runResult);
|
|
}
|
|
|
|
sb.append("### Results\n\n");
|
|
for (Map.Entry<Mode, List<RunResult>> entry : byMode.entrySet()) {
|
|
Mode mode = entry.getKey();
|
|
List<RunResult> sorted = new ArrayList<>(entry.getValue());
|
|
boolean lowerIsBetter = mode == Mode.AverageTime || mode == Mode.SampleTime || mode == Mode.SingleShotTime;
|
|
|
|
sorted.sort((a, b) -> {
|
|
double scoreA = a.getPrimaryResult().getScore();
|
|
double scoreB = b.getPrimaryResult().getScore();
|
|
return lowerIsBetter ? Double.compare(scoreA, scoreB) : Double.compare(scoreB, scoreA);
|
|
});
|
|
|
|
sb.append("#### ").append(mode).append("\n\n");
|
|
sb.append("| Rank | Fixture | Score | Error | Unit |\n");
|
|
sb.append("|---:|---|---:|---:|---|\n");
|
|
|
|
int rank = 1;
|
|
for (RunResult runResult : sorted) {
|
|
Result<?> result = runResult.getPrimaryResult();
|
|
String fixtureType = runResult.getParams().getParam("fixtureType");
|
|
|
|
sb.append("| ")
|
|
.append(rank++)
|
|
.append(" | `")
|
|
.append(fixtureType)
|
|
.append("` | ")
|
|
.append(formatDouble(result.getScore()))
|
|
.append(" | ")
|
|
.append(formatDouble(result.getScoreError()))
|
|
.append(" | `")
|
|
.append(result.getScoreUnit())
|
|
.append("` |\n");
|
|
}
|
|
|
|
Set<String> presentFixtures = new HashSet<>();
|
|
for (RunResult runResult : sorted) {
|
|
presentFixtures.add(runResult.getParams().getParam("fixtureType"));
|
|
}
|
|
|
|
for (String expectedFixture : expectedFixtureTypes()) {
|
|
if (presentFixtures.contains(expectedFixture)) {
|
|
continue;
|
|
}
|
|
sb.append("| - | `")
|
|
.append(expectedFixture)
|
|
.append("` | FAILED | - | `n/a` |\n");
|
|
}
|
|
sb.append("\n");
|
|
}
|
|
|
|
return sb.toString().trim();
|
|
}
|
|
|
|
private static List<String> expectedFixtureTypes() {
|
|
try {
|
|
java.lang.reflect.Field fixtureField = DataAccessBenchmark.class.getDeclaredField("fixtureType");
|
|
Param param = fixtureField.getAnnotation(Param.class);
|
|
if (param == null || param.value() == null || param.value().length == 0) {
|
|
return List.of();
|
|
}
|
|
return Arrays.asList(param.value());
|
|
} catch (NoSuchFieldException e) {
|
|
return List.of();
|
|
}
|
|
}
|
|
|
|
private static String formatDouble(double value) {
|
|
if (Double.isNaN(value)) {
|
|
return "NaN";
|
|
}
|
|
if (Double.isInfinite(value)) {
|
|
return value > 0 ? "+Inf" : "-Inf";
|
|
}
|
|
return String.format(Locale.US, "%.3f", value);
|
|
}
|
|
|
|
private static String detectCpuModel() {
|
|
String modelFromProperty = System.getProperty("cpu.model");
|
|
if (modelFromProperty != null && !modelFromProperty.isBlank()) {
|
|
return modelFromProperty.trim();
|
|
}
|
|
|
|
Path cpuInfo = Paths.get("/proc/cpuinfo");
|
|
if (Files.exists(cpuInfo)) {
|
|
try {
|
|
for (String line : Files.readAllLines(cpuInfo, StandardCharsets.UTF_8)) {
|
|
if (line.startsWith("model name")) {
|
|
int idx = line.indexOf(':');
|
|
if (idx >= 0 && idx + 1 < line.length()) {
|
|
String value = line.substring(idx + 1).trim();
|
|
if (!value.isBlank()) {
|
|
return value;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (IOException ignored) {
|
|
// Best effort only.
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static String detectRamModel() {
|
|
String modelFromProperty = System.getProperty("ram.model");
|
|
if (modelFromProperty != null && !modelFromProperty.isBlank()) {
|
|
return modelFromProperty.trim();
|
|
}
|
|
|
|
String modelFromEnv = System.getenv("RAM_MODEL");
|
|
if (modelFromEnv != null && !modelFromEnv.isBlank()) {
|
|
return modelFromEnv.trim();
|
|
}
|
|
|
|
String osName = System.getProperty("os.name", "").toLowerCase(Locale.ROOT);
|
|
if (osName.contains("linux")) {
|
|
String fromDmidecode = runLinuxCommandForSingleLine(
|
|
"bash", "-lc",
|
|
"if command -v dmidecode >/dev/null 2>&1; then "
|
|
+ "dmidecode -t memory 2>/dev/null | "
|
|
+ "awk -F: '/^[[:space:]]*Part Number:/{gsub(/^[ \\t]+|[ \\t]+$/, \"\", $2); "
|
|
+ "if($2!=\"\" && $2!=\"Not Specified\") {print $2; exit}}'; "
|
|
+ "fi"
|
|
);
|
|
if (fromDmidecode != null && !fromDmidecode.isBlank()) {
|
|
return fromDmidecode;
|
|
}
|
|
|
|
String fromLshw = runLinuxCommandForSingleLine(
|
|
"bash", "-lc",
|
|
"if command -v lshw >/dev/null 2>&1; then "
|
|
+ "lshw -class memory 2>/dev/null | "
|
|
+ "awk -F: '/^[[:space:]]*product:/{gsub(/^[ \\t]+|[ \\t]+$/, \"\", $2); "
|
|
+ "if($2!=\"\" && $2!=\"[empty]\") {print $2; exit}}'; "
|
|
+ "fi"
|
|
);
|
|
if (fromLshw != null && !fromLshw.isBlank()) {
|
|
return fromLshw;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static String runLinuxCommandForSingleLine(String... command) {
|
|
Process process = null;
|
|
try {
|
|
process = new ProcessBuilder(command)
|
|
.redirectErrorStream(true)
|
|
.start();
|
|
|
|
boolean finished = process.waitFor(3, TimeUnit.SECONDS);
|
|
if (!finished) {
|
|
process.destroyForcibly();
|
|
return null;
|
|
}
|
|
|
|
String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim();
|
|
if (output.isBlank()) {
|
|
return null;
|
|
}
|
|
|
|
int newlineIdx = output.indexOf('\n');
|
|
String firstLine = newlineIdx >= 0 ? output.substring(0, newlineIdx).trim() : output;
|
|
return firstLine.isBlank() ? null : firstLine;
|
|
} catch (Exception ignored) {
|
|
return null;
|
|
} finally {
|
|
if (process != null) {
|
|
process.destroy();
|
|
}
|
|
}
|
|
}
|
|
|
|
private static long detectTotalPhysicalMemoryBytes() {
|
|
try {
|
|
java.lang.management.OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
|
|
if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) {
|
|
return sunOsBean.getTotalMemorySize();
|
|
}
|
|
} catch (Throwable ignored) {
|
|
// Best effort only.
|
|
}
|
|
return -1;
|
|
}
|
|
}
|