From 8be0468d77cc5ab1dd77db0aa641b2722df04bd2 Mon Sep 17 00:00:00 2001 From: Lukas Tenbrink Date: Wed, 25 Jun 2025 20:17:14 +0200 Subject: [PATCH] Add graphs for binary size, build time, build memory use, startup time, and memory use (#111) - Change the storage type for extra information to the same as the other benchmarks. --- run-benchmarks.sh | 74 ++++++++++---------- web/layouts/benchmark/single.html | 109 +++++++++++++++++------------- web/src-data/graphs.json | 25 +++++++ 3 files changed, 126 insertions(+), 82 deletions(-) diff --git a/run-benchmarks.sh b/run-benchmarks.sh index d58f7c0..f4b5a42 100755 --- a/run-benchmarks.sh +++ b/run-benchmarks.sh @@ -194,6 +194,43 @@ $GODOT_RELEASE --audio-driver Dummy --gpu-index 1 -- --run-benchmarks --include- $GODOT_RELEASE --audio-driver Dummy --gpu-index 0 -- --run-benchmarks --include-benchmarks="rendering/*" --save-json="/tmp/intel.md" --json-results-prefix="intel" $GODOT_RELEASE --audio-driver Dummy --gpu-index 2 -- --run-benchmarks --include-benchmarks="rendering/*" --save-json="/tmp/nvidia.md" --json-results-prefix="nvidia" +echo "Appending extra benchmarks." +EXTRA_JSON=$(cat << EOF +"benchmarks": [ + { + "category": "Extra/Size", + "name": "Binary Size", + "results": {"cpu_debug": {"size_bytes":$BINARY_SIZE_DEBUG}, "cpu_release": {"size_bytes": $BINARY_SIZE_RELEASE}} + }, { + "category": "Extra/Build Time", + "name": "Build Time", + "results": {"cpu_debug": {"time":$TIME_TO_BUILD_DEBUG}, "cpu_release": {"time": $TIME_TO_BUILD_RELEASE}} + }, { + "category": "Extra/Build Memory Use", + "name": "Build Peak Memory Use", + "results": {"cpu_debug": {"ram_bytes":$PEAK_MEMORY_BUILD_DEBUG}, "cpu_release": {"ram_bytes": $PEAK_MEMORY_BUILD_RELEASE}} + }, { + "category": "Extra/Startup Time", + "name": "Startup + Shutdown Time", + "results": {"cpu_debug": {"time":$TIME_TO_STARTUP_SHUTDOWN_DEBUG}, "cpu_release": {"time": $TIME_TO_STARTUP_SHUTDOWN_RELEASE}} + }, { + "category": "Extra/Startup Time", + "name": "Startup Time With Shader Cache", + "results": {"cpu_debug": {"time":$TIME_TO_STARTUP_SHADER_CACHE}, "cpu_release": {"time": $TIME_TO_STARTUP_SHADER_CACHE}} + }, { + "category": "Extra/Startup Time", + "name": "Startup Time Without Shader Cache", + "results": {"cpu_debug": {"time":$TIME_TO_STARTUP_NO_SHADER_CACHE}, "cpu_release": {"time": $TIME_TO_STARTUP_NO_SHADER_CACHE}} + }, { + "category": "Extra/Memory Use", + "name": "Startup + Shutdown Peak Memory Use", + "results": {"cpu_debug": {"ram_bytes":$PEAK_MEMORY_STARTUP_SHUTDOWN_DEBUG}, "cpu_release": {"ram_bytes": $PEAK_MEMORY_STARTUP_SHUTDOWN_RELEASE}} + } +] +EOF +) +echo "$EXTRA_JSON" > "/tmp/extra.md" + # We cloned a copy of the repository above so we can push the new JSON files to it. # The website build is performed by GitHub Actions on the `main` branch of the repository below, # so we only push files to it and do nothing else. @@ -202,7 +239,7 @@ cd /tmp/godot-benchmarks-results/ # Merge benchmark run JSONs together. # Use editor build as release build errors due to missing PCK file. echo "Merging JSON files together." -$GODOT_DEBUG --headless --path "$DIR" --script merge_json.gd -- /tmp/cpu_debug.md /tmp/cpu_release.md /tmp/amd.md /tmp/intel.md /tmp/nvidia.md --output-path /tmp/merged.md +$GODOT_DEBUG --headless --path "$DIR" --script merge_json.gd -- /tmp/cpu_debug.md /tmp/cpu_release.md /tmp/amd.md /tmp/intel.md /tmp/nvidia.md /tmp/extra.md --output-path /tmp/merged.md OUTPUT_PATH="/tmp/godot-benchmarks-results/${DATE}_${COMMIT_HASH}.md" rm -f "$OUTPUT_PATH" @@ -215,41 +252,6 @@ strip "$GODOT_DEBUG" "$GODOT_RELEASE" BINARY_SIZE_DEBUG="$(stat --printf="%s" "$GODOT_DEBUG")" BINARY_SIZE_RELEASE="$(stat --printf="%s" "$GODOT_RELEASE")" -# Add extra JSON at the end of the merged JSON. We assume the merged JSON has no -# newline at the end of file, as Godot writes it. To append more data to the -# JSON dictionary, we remove the last `}` character and add a `,` instead. -echo "Appending extra JSON at the end of the merged JSON." -EXTRA_JSON=$(cat << EOF -"binary_size": { - "debug": $BINARY_SIZE_DEBUG, - "release": $BINARY_SIZE_RELEASE -}, -"build_time": { - "debug": $TIME_TO_BUILD_DEBUG, - "release": $TIME_TO_BUILD_RELEASE -}, -"build_peak_memory_usage": { - "debug": $PEAK_MEMORY_BUILD_DEBUG, - "release": $PEAK_MEMORY_BUILD_RELEASE -}, -"empty_project_startup_shutdown_time": { - "debug": $TIME_TO_STARTUP_SHUTDOWN_DEBUG, - "release": $TIME_TO_STARTUP_SHUTDOWN_RELEASE -}, -"empty_project_startup_shutdown_peak_memory_usage": { - "debug": $PEAK_MEMORY_STARTUP_SHUTDOWN_DEBUG, - "release": $PEAK_MEMORY_STARTUP_SHUTDOWN_RELEASE -}, -"empty_project_editor_startup_shader_cache": { - "debug": $TIME_TO_STARTUP_SHADER_CACHE -}, -"empty_project_editor_startup_no_shader_cache": { - "debug": $TIME_TO_STARTUP_NO_SHADER_CACHE -} -EOF -) -echo "$(head -c -1 /tmp/merged.md),$EXTRA_JSON}" > "$OUTPUT_PATH" - # Build website files after running all benchmarks, so that benchmarks # appear on the web interface. echo "Pushing results to godot-benchmarks repository." diff --git a/web/layouts/benchmark/single.html b/web/layouts/benchmark/single.html index 6f73744..5a1dc49 100644 --- a/web/layouts/benchmark/single.html +++ b/web/layouts/benchmark/single.html @@ -48,52 +48,6 @@ -

Engine information

- - - - - - - - - - - - - - - - - - - - - - - - - -
Build type - Debug Debug editor
 - Release Release export template -
Time to build - Debug {{ mul $benchmark.build_time.debug 0.001 | lang.FormatNumber 0 }} seconds
 - Release {{ mul $benchmark.build_time.release 0.001 | lang.FormatNumber 0 }} seconds -
Build peak memory usage - Debug {{ mul $benchmark.build_peak_memory_usage.debug 0.001 | lang.FormatNumber 2 }} MB
 - Release {{ mul $benchmark.build_peak_memory_usage.release 0.001 | lang.FormatNumber 2 }} MB -
Startup + shutdown time - Debug {{ $benchmark.empty_project_startup_shutdown_time.debug | lang.FormatNumber 0 }} ms
 - Release {{ $benchmark.empty_project_startup_shutdown_time.release | lang.FormatNumber 0 }} ms -
Startup + shutdown peak memory usage - Debug {{ mul $benchmark.empty_project_startup_shutdown_peak_memory_usage.debug 0.001 | lang.FormatNumber 2 }} MB
 - Release {{ mul $benchmark.empty_project_startup_shutdown_peak_memory_usage.release 0.001 | lang.FormatNumber 2 }} MB -
Binary size - Debug {{ mul $benchmark.binary_size.debug 0.001 | lang.FormatNumber 0 }} KB
 - Release {{ mul $benchmark.binary_size.release 0.001 | lang.FormatNumber 0 }} KB -
-

Benchmark results

For all values, lower is better.
@@ -201,4 +155,67 @@
+
+ RAM + + + + + + + + + {{ range $benchmark.benchmarks }} + {{ if gt .results.cpu_debug.ram_bytes 0 }} + + + + + {{ end }} + {{ end }} + +
NameRAM
+ + {{ delimit (first (sub (len .path) 1) .path) " > "}} + +
+ {{ index (last 1 .path) 0 }} + 		+ Debug {{ .results.cpu_debug.ram_bytes }} b
 + Release {{ .results.cpu_release.ram_bytes }} b +
+
+ +
+ Size + + + + + + + + + {{ range $benchmark.benchmarks }} + {{ if gt .results.cpu_debug.size_bytes 0 }} + + + + + {{ end }} + {{ end }} + +
NameSize
+ + {{ delimit (first (sub (len .path) 1) .path) " > "}} + +
+ {{ index (last 1 .path) 0 }} + 		+ Debug {{ .results.cpu_debug.size_bytes }} b
 + Release {{ .results.cpu_release.size_bytes }} b +
+
+ + {{ end }} diff --git a/web/src-data/graphs.json b/web/src-data/graphs.json index cf95aac..c37a732 100644 --- a/web/src-data/graphs.json +++ b/web/src-data/graphs.json @@ -148,5 +148,30 @@ "id": "viewport-allocation", "title": "Viewport Allocation", "benchmark-path-prefix": "Viewport/Allocation" + }, + { + "id": "binary-size", + "title": "Size", + "benchmark-path-prefix": "Extra/Size" + }, + { + "id": "build-time", + "title": "Build Time", + "benchmark-path-prefix": "Extra/Build Time" + }, + { + "id": "build-memory-use", + "title": "Build Memory Use", + "benchmark-path-prefix": "Extra/Build Memory Use" + }, + { + "id": "startup-time", + "title": "Startup Time", + "benchmark-path-prefix": "Extra/Startup Time" + }, + { + "id": "memory-use", + "title": "Memory Use", + "benchmark-path-prefix": "Extra/Memory Use" } ]