Add a script to automatically generate and upload release

The script must be run from the godot-build-scripts directory
used to build Godot. It requires access to both folders to
read build artifacts from there and create commits here.

.gitignore

@@ -1,2 +1,2 @@
 # Project folders.
-temp/
+tmp/

tools/bootstrap/generate-metadata.py

@@ -3,7 +3,7 @@
 
 ### Generate JSON metadata files for each official release of Godot.
 ###
-### Files are put into a temporary folder temp/releases. To generate
+### Files are put into a temporary folder tmp/releases. To generate
 ### the data we extract dates and commit hashes from releases published
 ### on TuxFamily. We also extract SHA512 checksums for release files
 ### where possible.
@@ -107,9 +107,9 @@ def generate_file(version_name, release_name, release_status, release_url):
 
     # Open the file for writing.
 
-    output_path = f"./temp/releases/godot-{release_name}.json"
+    output_path = f"./tmp/releases/godot-{release_name}.json"
     if release_status == "stable":
-        output_path = f"./temp/releases/godot-{release_name}-stable.json"
+        output_path = f"./tmp/releases/godot-{release_name}-stable.json"
 
     with open(output_path, 'w') as f:
         # Get the commit hash / git reference.
@@ -190,8 +190,8 @@ for match in matches:
         version_names.append(subfolder_name[:-1])
 
 # Create the output directory if it doesn't exist.
-if not os.path.exists("./temp/releases"):
+if not os.path.exists("./tmp/releases"):
-    os.makedirs("./temp/releases")
+    os.makedirs("./tmp/releases")
 
 for version_name in version_names:
     version_url = url + version_name

tools/bootstrap/generate-releases.py

@@ -8,7 +8,7 @@
 ### in the linked repository. Make sure to use gh to configure
 ### the default repository for this project's folder.
 ###
-### Generated release notes are available in temp/notes for examination.
+### Generated release notes are available in tmp/notes for examination.
 
 
 import json
@@ -55,7 +55,6 @@ def generate_notes(release_data):
     version_description = ""
 
     if version_status == "stable":
-        version_bits = version_version.split(".")
         if version_flavor == "major":
             version_description = "a major release introducing new features and considerable changes to core systems. **Major version releases contain compatibility breaking changes.**"
         elif version_flavor == "minor":
@@ -137,7 +136,7 @@ def generate_notes(release_data):
     return notes
 
 
-with open("./temp/versions.yml", "r") as f:
+with open("./tmp/versions.yml", "r") as f:
     try:
         website_versions = yaml.safe_load(f)
     except yaml.YAMLError as exc:
@@ -171,8 +170,8 @@ releases.sort(key=lambda x: x['data']['release_date'])
 # match the release date.
 
 # Create the output directory if it doesn't exist.
-if not os.path.exists("./temp/notes"):
+if not os.path.exists("./tmp/notes"):
-    os.makedirs("./temp/notes")
+    os.makedirs("./tmp/notes")
 
 for release_data in releases:
     release_tag = f"{release_data['data']['version']}-{release_data['data']['status']}"
@@ -182,7 +181,7 @@ for release_data in releases:
         prerelease_flag = "--prerelease"
 
     release_notes = generate_notes(release_data['data'])
-    release_notes_file = f"./temp/notes/release-notes-{release_tag}.txt"
+    release_notes_file = f"./tmp/notes/release-notes-{release_tag}.txt"
     with open(release_notes_file, 'w') as temp_notes:
         temp_notes.write(release_notes)
 

tools/create-release-metadata.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+from datetime import datetime
+
+
+def find_file_checksums(release_path):
+    files = []
+
+    checksums_path = f"{release_path}/SHA512-SUMS.txt"
+    with open(checksums_path, 'r') as checksums:
+        for line in checksums:
+            split_line = line.split("  ")
+            files.append({
+                "filename": split_line[1].strip(),
+                "checksum": split_line[0].strip()
+            })
+
+    return files
+
+
+def generate_file(version_version: str, version_status: str, git_reference: str):
+    # Open the file for writing.
+
+    basedir = os.environ.get("basedir")
+    buildsdir = os.environ.get('buildsdir')
+
+    output_path = f"{buildsdir}/releases/godot-{version_version}-{version_status}.json"
+    with open(output_path, 'w') as f:
+        release_name = version_version
+        commit_hash = git_reference
+        if version_status == "stable":
+            commit_hash = f"{version_version}-stable"
+        else:
+            release_name = f"{version_version}-{version_status}"
+
+        # Start writing the file with basic meta information.
+        f.write(
+            f'{{\n'
+            f'    "name": "{release_name}",\n'
+            f'    "version": "{version_version}",\n'
+            f'    "status": "{version_status}",\n'
+            f'    "release_date": {int(datetime.now().timestamp())},\n'
+            f'    "git_reference": "{commit_hash}",\n'
+            f'\n'
+            f'    "files": [\n'
+        )
+
+        # Generate the list of files.
+
+        release_folder = f"{basedir}/releases/{version_version}-{version_status}"
+        standard_files = find_file_checksums(f"{release_folder}")
+        mono_files = find_file_checksums(f"{release_folder}/mono")
+
+        for i, file in enumerate(standard_files):
+            f.write(
+                f'        {{\n'
+                f'            "filename": "{file["filename"]}",\n'
+                f'            "checksum": "{file["checksum"]}"\n'
+                f'        }}{"" if i == len(standard_files) - 1 and len(mono_files) == 0 else ","}\n'
+            )
+
+        for i, file in enumerate(mono_files):
+            f.write(
+                f'        {{\n'
+                f'            "filename": "{file["filename"]}",\n'
+                f'            "checksum": "{file["checksum"]}"\n'
+                f'        }}{"" if i == len(mono_files) - 1 else ","}\n'
+            )
+
+        # Finish the file.
+        f.write(
+            f'    ]\n'
+            f'}}\n'
+        )
+
+        print(f"Written release metadata to '{output_path}'.")
+
+
+def main() -> None:
+    if os.environ.get("basedir") == "" or os.environ.get("buildsdir") == "":
+        print("Failed to create release metadata: Missing 'basedir' (godot-build-scripts) and 'buildsdir' (godot-builds) environment variables.\n")
+        exit(1)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", default="", help="Godot version in the major.minor.patch format (patch should be omitted for major and minor releases).")
+    parser.add_argument("-f", "--flavor", default="stable", help="Release flavor, e.g. dev, alpha, beta, rc, stable (defaults to stable).")
+    parser.add_argument("-g", "--git", default="", help="Git commit hash tagged for this release.")
+    args = parser.parse_args()
+
+    if args.version == "" or args.git == "":
+        print("Failed to create release metadata: Godot version and git hash cannot be empty.\n")
+        parser.print_help()
+        exit(1)
+
+    release_version = args.version
+    release_flavor = args.flavor
+    if release_flavor == "":
+        release_flavor = "stable"
+
+    generate_file(release_version, release_flavor, args.git)
+
+
+if __name__ == "__main__":
+    main()

tools/create-release-notes.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+
+def get_version_name(version_version: str, version_status: str) -> str:
+    version_name = version_version
+
+    if version_status == "stable":
+        return version_name
+
+    version_name += " "
+    if version_status.startswith("rc"):
+        version_name += f"RC {version_status.removeprefix('rc')}"
+    elif version_status.startswith("beta"):
+        version_name += f"beta {version_status.removeprefix('beta')}"
+    elif version_status.startswith("alpha"):
+        version_name += f"alpha {version_status.removeprefix('alpha')}"
+    elif version_status.startswith("dev"):
+        version_name += f"dev {version_status.removeprefix('dev')}"
+    else:
+        version_name += version_status
+
+    return version_name
+
+
+def get_version_description(version_version: str, version_status: str, version_flavor: str) -> str:
+    version_description = ""
+
+    if version_status == "stable":
+        if version_flavor == "major":
+            version_description = "a major release introducing new features and considerable changes to core systems. **Major version releases contain compatibility breaking changes.**"
+        elif version_flavor == "minor":
+            version_description = "a feature release improving upon the previous version in many aspects, such as usability and performance. Feature releases also contain new features, but preserve compatibility with previous releases."
+        else:
+            version_description = "a maintenance release addressing stability and usability issues, and fixing all sorts of bugs. Maintenance releases are compatible with previous releases and are recommended for adoption."
+    else:
+        flavor_name = "maintenance"
+        if version_flavor == "major":
+            flavor_name = "major"
+        elif version_flavor == "minor":
+            flavor_name = "feature"
+
+        if version_status.startswith("rc"):
+            version_description = f"a release candidate for the {version_version} {flavor_name} release. Release candidates focus on finalizing the release and fixing remaining critical bugs."
+        elif version_status.startswith("beta"):
+            version_description = f"a beta snapshot for the {version_version} {flavor_name} release. Beta snapshots are feature-complete and provided for public beta testing to catch as many bugs as possible ahead of the stable release."
+        else: # alphas and devs go here.
+            version_description = f"a dev snapshot for the {version_version} {flavor_name} release. Dev snapshots are in-development builds of the engine provided for early testing and feature evaluation while the engine is still being worked on."
+
+    return version_description
+
+
+def get_release_notes_url(version_version: str, version_status: str, version_flavor: str) -> str:
+    release_notes_slug = ""
+    version_sluggified = version_version.replace(".", "-")
+
+    if version_status == "stable":
+        if version_flavor == "major":
+            release_notes_slug = f"major-release-godot-{version_sluggified}"
+        elif version_flavor == "minor":
+            release_notes_slug = f"feature-release-godot-{version_sluggified}"
+        else:
+            release_notes_slug = f"maintenance-release-godot-{version_sluggified}"
+    else:
+        if version_status.startswith("rc"):
+            status_sluggified = version_status.removeprefix("rc").replace(".", "-")
+            release_notes_slug = f"release-candidate-godot-{version_sluggified}-rc-{status_sluggified}"
+        elif version_status.startswith("beta"):
+            status_sluggified = version_status.removeprefix("beta").replace(".", "-")
+            release_notes_slug = f"dev-snapshot-godot-{version_sluggified}-beta-{status_sluggified}"
+        elif version_status.startswith("alpha"):
+            status_sluggified = version_status.removeprefix("alpha").replace(".", "-")
+            release_notes_slug = f"dev-snapshot-godot-{version_sluggified}-alpha-{status_sluggified}"
+        elif version_status.startswith("dev"):
+            status_sluggified = version_status.removeprefix("dev").replace(".", "-")
+            release_notes_slug = f"dev-snapshot-godot-{version_sluggified}-dev-{status_sluggified}"
+        else:
+            status_sluggified = version_status.replace(".", "-")
+            release_notes_slug = f"dev-snapshot-godot-{version_sluggified}-{status_sluggified}"
+
+    return f"https://godotengine.org/article/{release_notes_slug}/"
+
+
+def generate_notes(version_version: str, version_status: str, git_reference: str) -> None:
+    notes = ""
+
+    version_tag = f"{version_version}-{version_status}"
+
+    version_bits = version_version.split(".")
+    version_flavor = "patch"
+    if len(version_bits) == 2 and version_bits[1] == "0":
+        version_flavor = "major"
+    elif len(version_bits) == 2 and version_bits[1] != "0":
+        version_flavor = "minor"
+
+    # Add the intro line.
+
+    version_name = get_version_name(version_version, version_status)
+    version_description = get_version_description(version_version, version_status, version_flavor)
+
+    notes += f"**Godot {version_name}** is {version_description}\n\n"
+
+    # Link to the bug tracker.
+
+    notes += "Report bugs on GitHub after checking that they haven't been reported:\n"
+    notes += "- https://github.com/godotengine/godot/issues\n"
+    notes += "\n"
+
+    # Add build information.
+
+    # Only for pre-releases.
+    if version_status != "stable":
+        commit_hash = git_reference
+        notes += f"Built from commit [{commit_hash}](https://github.com/godotengine/godot/commit/{commit_hash}).\n"
+        notes += f"To make a custom build which would also be recognized as {version_status}, you should define `GODOT_VERSION_STATUS={version_status}` in your build environment prior to compiling.\n"
+        notes += "\n"
+
+    # Add useful links.
+
+    notes += "----\n"
+    notes += "\n"
+
+    release_notes_url = get_release_notes_url(version_version, version_status, version_flavor)
+
+    notes += f"- [Release notes]({release_notes_url})\n"
+
+    if version_status == "stable":
+        notes += f"- [Complete changelog](https://godotengine.github.io/godot-interactive-changelog/#{version_version})\n"
+        notes += f"- [Curated changelog](https://github.com/godotengine/godot/blob/{version_tag}/CHANGELOG.md)\n"
+    else:
+        notes += f"- [Complete changelog](https://godotengine.github.io/godot-interactive-changelog/#{version_tag})\n"
+
+    notes += "- Download (GitHub): Expand **Assets** below\n"
+
+    if version_status == "stable":
+        notes += f"- [Download (TuxFamily)](https://downloads.tuxfamily.org/godotengine/{version_version})\n"
+    else:
+        notes += f"- [Download (TuxFamily)](https://downloads.tuxfamily.org/godotengine/{version_version}/{version_status})\n"
+
+    notes += "\n"
+    notes += "*All files for this release are mirrored under **Assets** below.*\n"
+
+    return notes
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", default="", help="Godot version in the major.minor.patch format (patch should be omitted for major and minor releases).")
+    parser.add_argument("-f", "--flavor", default="stable", help="Release flavor, e.g. dev, alpha, beta, rc, stable (defaults to stable).")
+    parser.add_argument("-g", "--git", default="", help="Git commit hash tagged for this release.")
+    args = parser.parse_args()
+
+    if args.version == "" or args.git == "":
+        print("Failed to create release notes: Godot version and git hash cannot be empty.\n")
+        parser.print_help()
+        exit(1)
+
+    release_version = args.version
+    release_flavor = args.flavor
+    if release_flavor == "":
+        release_flavor = "stable"
+    release_tag = f"{release_version}-{release_flavor}"
+
+    release_notes = generate_notes(release_version, release_flavor, args.git)
+    release_notes_file = f"./tmp/release-notes-{release_tag}.txt"
+    with open(release_notes_file, 'w') as temp_notes:
+        temp_notes.write(release_notes)
+
+    print(f"Written release notes to '{release_notes_file}'.")
+
+
+if __name__ == "__main__":
+    main()

tools/upload-github.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+set -e
+
+# Generate GitHub release for a Godot version and upload artifacts.
+#
+# Usage: ./upload-github.sh -v 3.6
+# Usage: ./upload-github.sh -v 3.6 -f beta3
+# Usage: ./upload-github.sh -v 3.6 -f beta3 -r owner/repository
+#
+# Run this script from the root of the godot-build-scripts folder
+# after building Godot.
+
+# Folder this script is called from, a.k.a its working directory.
+export basedir=$(pwd)
+
+# Folder where this scripts resides in.
+scriptpath=$(readlink -f "$0")
+scriptdir=$(dirname "$scriptpath")
+# Root folder of this project, hopefully.
+export buildsdir=$(dirname "$scriptdir")
+
+if [ ! -d "${basedir}/releases" ] || [ ! -d "${basedir}/tmp" ]; then
+  echo "Cannot find one of the required folders: releases, tmp."
+  echo "  Make sure you're running this script from the root of your godot-build-scripts clone, and that Godot has been built with it."
+  exit 1
+fi
+
+# Setup.
+
+godot_version=""
+godot_flavor="stable"
+godot_repository="godotengine/godot-builds"
+
+while getopts "v:f:r:" opt; do
+  case "$opt" in
+  v)
+    godot_version=$OPTARG
+    ;;
+  f)
+    godot_flavor=$OPTARG
+    ;;
+  r)
+    godot_repository=$OPTARG
+    ;;
+  esac
+done
+
+release_tag="$godot_version-$godot_flavor"
+
+echo "Preparing release $release_tag..."
+
+version_path="$basedir/releases/$release_tag"
+if [ ! -d "${version_path}" ]; then
+  echo "Cannot find the release folder at $version_path."
+  echo "  Make sure you're running this script from the root of godot-build-scripts, and that Godot has been built."
+  exit 1
+fi
+
+cd git
+git_reference=$(git rev-parse HEAD)
+cd ..
+
+# Generate release metadata and commit it to Git.
+
+echo "Creating and committing release metadata for $release_tag..."
+
+if ! $buildsdir/tools/create-release-metadata.py -v $godot_version -f $godot_flavor -g $git_reference; then
+  echo "Failed to create release metadata for $release_tag."
+  exit 1
+fi
+
+cd $buildsdir
+git add ./releases/godot-$release_tag.json
+git commit -m "Add Godot $release_tag"
+git tag $release_tag
+if ! git push --atomic origin release-automation $release_tag; then
+  echo "Failed to push release metadata for $release_tag to GitHub."
+  exit 1
+fi
+cd $basedir
+
+# Exactly one time it failed to create release immediately after pushing the tag,
+# so we use this for protection...
+sleep 2
+
+# Generate GitHub release.
+
+echo "Creating and publishing GitHub release for $release_tag..."
+
+if ! $buildsdir/tools/create-release-notes.py -v $godot_version -f $godot_flavor -g $git_reference; then
+  echo "Failed to create release notes for $release_tag."
+  exit 1
+fi
+
+release_notes="$basedir/tmp/release-notes-$release_tag.txt"
+release_flag=""
+if [ $godot_flavor != "stable" ]; then
+  release_flag="--prerelease"
+fi
+
+if ! gh release create $release_tag --verify-tag --title "$release_tag" --notes-file $release_notes $release_flag -R $godot_repository; then
+  echo "Cannot create a GitHub release for $release_tag."
+  exit 1
+fi
+
+# Upload release files to GitHub.
+
+echo "Uploading release files from $version_path..."
+
+# We are picking up all relevant files lazily, using a substring.
+# The first letter can be in either case, so we're skipping it.
+for f in $version_path/*odot*; do
+  echo "Uploading $f..."
+  gh release upload $release_tag $f -R $godot_repository
+done
+
+# Do the same for .NET builds.
+for f in $version_path/mono/*odot*; do
+  echo "Uploading $f..."
+  gh release upload $release_tag $f -R $godot_repository
+done
+
+# README.txt is only generated for pre-releases.
+readme_path="$version_path/README.txt"
+if [ $godot_flavor != "stable" ] && [ -f "${readme_path}" ]; then
+  echo "Uploading $readme_path..."
+  gh release upload $release_tag $readme_path -R $godot_repository
+fi
+
+# SHA512-SUMS.txt is split into two: classic and mono, and we need to upload them as one.
+checksums_path="$basedir/tmp/SHA512-SUMS.txt"
+cp $basedir/releases/$release_tag/SHA512-SUMS.txt $checksums_path
+if [ -d "${basedir}/releases/${release_tag}/mono" ]; then
+  cat $basedir/releases/$release_tag/mono/SHA512-SUMS.txt >> $checksums_path
+fi
+
+echo "Uploading $checksums_path..."
+gh release upload $release_tag $checksums_path -R $godot_repository
+
+echo "Done."