Add configs and data for Godot 3.0

- Give more chances to finish downloading bigger commit sets

compose-db.js

@@ -18,6 +18,7 @@ const LogFormat = {
 
 const COMMITS_PER_PAGE = 150;
 const API_DELAY_MSEC = 2500;
+const API_MAX_RETRIES = 5;
 const API_RATE_LIMIT = `
   rateLimit {
     limit
@@ -28,6 +29,8 @@ const API_RATE_LIMIT = `
   }
 `;
 
+const EXEC_MAX_BUFFER = 1024 * 1024 * 32;
+
 const GIT_HEAD_COMMIT_RE = RegExp("^commit ([a-zA-Z0-9-_]+)$");
 const GIT_HEAD_MERGE_RE = RegExp("^Merge: (.+) (.+)$");
 const GIT_HEAD_AUTHOR_RE = RegExp("^Author: (.+)$");
@@ -94,7 +97,7 @@ class DataFetcher {
             await clearDir("./temp");
 
             // Checkout a shallow clone of the repository; we are only interested in its history.
-            await exec(`git clone --filter=tree:0 --branch ${atCommit} --single-branch ${this.repo_ssh_path}`, { cwd: "./temp" });
+            await exec(`git clone --filter=tree:0 --branch ${atCommit} --single-branch ${this.repo_ssh_path}`, { cwd: "./temp", maxBuffer: EXEC_MAX_BUFFER });
         } catch (err) {
             console.error("    Error checking out a copy of the target repository: " + err);
             process.exitCode = ExitCodes.ExecFailure;
@@ -104,7 +107,7 @@ class DataFetcher {
 
     async countCommitHistory(fromCommit, toCommit) {
         try {
-            const { stdout, stderr } = await exec(`git log --pretty=oneline ${fromCommit}..${toCommit}`, { cwd: `./temp/${this.data_repo}` });
+            const { stdout, stderr } = await exec(`git log --pretty=oneline ${fromCommit}..${toCommit}`, { cwd: `./temp/${this.data_repo}`, maxBuffer: EXEC_MAX_BUFFER });
 
             const commitHistory = stdout.trimEnd();
             await this._logResponse(commitHistory, "_commit_shortlog", LogFormat.Raw);
@@ -118,7 +121,7 @@ class DataFetcher {
 
     async getCommitHistory(fromCommit, toCommit) {
         try {
-            const { stdout, stderr } = await exec(`git log --pretty=full ${fromCommit}..${toCommit}`, { cwd: `./temp/${this.data_repo}` });
+            const { stdout, stderr } = await exec(`git log --pretty=full ${fromCommit}..${toCommit}`, { cwd: `./temp/${this.data_repo}`, maxBuffer: EXEC_MAX_BUFFER });
 
             const commitHistory = stdout;
             await this._logResponse(commitHistory, "_commit_history", LogFormat.Raw);
@@ -287,7 +290,7 @@ class DataFetcher {
 
             console.log(`    Requesting batch ${page}/${totalPages} of commit and pull request data.`);
 
-            const res = await this.fetchGithub(query, 2);
+            const res = await this.fetchGithub(query, API_MAX_RETRIES);
             if (res.status !== 200) {
                 this._handleResponseErrors(this.api_repository_id, res);
                 process.exitCode = ExitCodes.RequestFailure;
@@ -543,7 +546,7 @@ class DataProcessor {
 
                 const pullsRaw = mapNodes(item.associatedPullRequests)
                     .filter((pullData) => {
-                        return pullData.baseRef.repository.nameWithOwner === targetRepo;
+                        return pullData.baseRef && pullData.baseRef.repository.nameWithOwner === targetRepo;
                     });
                 if (pullsRaw.length === 0) {
                     continue;
@@ -742,6 +745,7 @@ async function main() {
     // Internal utility methods.
     const checkForExit = () => {
         if (process.exitCode > 0) {
+            console.log(`   Terminating with an exit code ${process.exitCode}.`);
             process.exit();
         }
     };

configs/godotengine.godot.3.0.1.json

@@ -0,0 +1,21 @@
+{
+    "name": "3.0.1",
+    "ref": "3.0.1-stable",
+    "from_ref": "3.0-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-3-0-1/",
+
+    "releases": [
+        {
+            "name": "rc1",
+            "ref": "370f84f41cf3ce493c2de687455880d9e1e04be6",
+            "from_ref": "3.0-stable",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-1-rc1/"
+        },
+        {
+            "name": "stable",
+            "ref": "3.0.1-stable",
+            "from_ref": "370f84f41cf3ce493c2de687455880d9e1e04be6",
+            "article": "https://godotengine.org/article/maintenance-release-godot-3-0-1/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.2.json

@@ -0,0 +1,15 @@
+{
+    "name": "3.0.2",
+    "ref": "3.0.2-stable",
+    "from_ref": "3.0.1-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-302/",
+
+    "releases": [
+        {
+            "name": "stable",
+            "ref": "3.0.2-stable",
+            "from_ref": "3.0.1-stable",
+            "article": "https://godotengine.org/article/maintenance-release-godot-302/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.3.json

@@ -0,0 +1,33 @@
+{
+    "name": "3.0.3",
+    "ref": "3.0.3-stable",
+    "from_ref": "3.0.2-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-3-0-3/",
+
+    "releases": [
+        {
+            "name": "rc1",
+            "ref": "63e70e3cd19d4ef42a293de40ffaf11aa735dad6",
+            "from_ref": "3.0.2-stable",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-3-rc-1/"
+        },
+        {
+            "name": "rc2",
+            "ref": "6635f2a1085a85f4195401d27d079a27bd98f3e0",
+            "from_ref": "63e70e3cd19d4ef42a293de40ffaf11aa735dad6",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-3-rc-2/"
+        },
+        {
+            "name": "rc3",
+            "ref": "f6406398670f41b55cd8e47bf5d8a1e764fb0c02",
+            "from_ref": "6635f2a1085a85f4195401d27d079a27bd98f3e0",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-3-rc-3/"
+        },
+        {
+            "name": "stable",
+            "ref": "3.0.3-stable",
+            "from_ref": "f6406398670f41b55cd8e47bf5d8a1e764fb0c02",
+            "article": "https://godotengine.org/article/maintenance-release-godot-3-0-3/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.4.json

@@ -0,0 +1,15 @@
+{
+    "name": "3.0.4",
+    "ref": "3.0.4-stable",
+    "from_ref": "3.0.3-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-3-0-4/",
+
+    "releases": [
+        {
+            "name": "stable",
+            "ref": "3.0.4-stable",
+            "from_ref": "3.0.3-stable",
+            "article": "https://godotengine.org/article/maintenance-release-godot-3-0-4/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.5.json

@@ -0,0 +1,15 @@
+{
+    "name": "3.0.5",
+    "ref": "3.0.5-stable",
+    "from_ref": "3.0.4-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-3-0-5/",
+
+    "releases": [
+        {
+            "name": "stable",
+            "ref": "3.0.5-stable",
+            "from_ref": "3.0.4-stable",
+            "article": "https://godotengine.org/article/maintenance-release-godot-3-0-5/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.6.json

@@ -0,0 +1,15 @@
+{
+    "name": "3.0.6",
+    "ref": "3.0.6-stable",
+    "from_ref": "3.0.5-stable",
+    "article": "https://godotengine.org/article/maintenance-release-godot-3-0-6/",
+
+    "releases": [
+        {
+            "name": "stable",
+            "ref": "3.0.6-stable",
+            "from_ref": "3.0.5-stable",
+            "article": "https://godotengine.org/article/maintenance-release-godot-3-0-6/"
+        }
+    ]
+}

configs/godotengine.godot.3.0.json

@@ -0,0 +1,57 @@
+{
+    "name": "3.0",
+    "ref": "3.0-stable",
+    "from_ref": "2.0-stable",
+    "article": "https://godotengine.org/article/godot-3-0-released/",
+
+    "releases": [
+        {
+            "name": "alpha1",
+            "ref": "be5adec2868983d2ec958e626ba451fbfba0b761",
+            "from_ref": "2.0-stable",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-alpha-1/"
+        },
+        {
+            "name": "alpha2",
+            "ref": "f021b0038940ecc03aa43721f03b3b3f283e00ed",
+            "from_ref": "be5adec2868983d2ec958e626ba451fbfba0b761",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-alpha-2/"
+        },
+        {
+            "name": "beta1",
+            "ref": "9725f33ac606ed4b26573015bcfdb86ed6741323",
+            "from_ref": "f021b0038940ecc03aa43721f03b3b3f283e00ed",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-beta-1/"
+        },
+        {
+            "name": "beta2",
+            "ref": "0b1e6ec2195a22ad66513685d0c0850eaeb9b528",
+            "from_ref": "9725f33ac606ed4b26573015bcfdb86ed6741323",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-beta-2/"
+        },
+        {
+            "name": "rc1",
+            "ref": "6a4521da2ae0b08f42001c2c38396b28a2549c14",
+            "from_ref": "0b1e6ec2195a22ad66513685d0c0850eaeb9b528",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-rc-1/"
+        },
+        {
+            "name": "rc2",
+            "ref": "fe2932a969cdc0483c31c12c1f8bfd5868401da8",
+            "from_ref": "6a4521da2ae0b08f42001c2c38396b28a2549c14",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-rc-2/"
+        },
+        {
+            "name": "rc3",
+            "ref": "59e83af201af5a93c7a13750d781c050c2275c07",
+            "from_ref": "fe2932a969cdc0483c31c12c1f8bfd5868401da8",
+            "article": "https://godotengine.org/article/dev-snapshot-godot-3-0-rc-3/"
+        },
+        {
+            "name": "stable",
+            "ref": "3.0-stable",
+            "from_ref": "59e83af201af5a93c7a13750d781c050c2275c07",
+            "article": "https://godotengine.org/article/godot-3-0-released/"
+        }
+    ]
+}