Merge pull request #5 from godotengine/compose-db-retry

compose-db: Add retry logic like in interactive changelog
2025-12-31 21:48:29 +03:00 · 2025-09-23 14:24:23 +02:00
parent 670c5b19fd 61511fc64b
commit 3be80247ef
4 changed files with 90 additions and 17 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,6 +3,7 @@ name: Continuous integration
 on:
  push:
    branches: [ master ]
+  pull_request:
  schedule:
    # Run every 15 minutes of every hour, starting at 5 minutes (5m, 20m, 35m, 50m).
    # The slight offset is there to try and avoid the high load times.
@@ -52,6 +53,7 @@ jobs:
        path: out

    - name: Deploy to GitHub Pages 🚀
+      if: github.event_name != 'pull_request'
      uses: JamesIves/github-pages-deploy-action@v4
      with:
        branch: gh-pages
--- a/compose-db.js
+++ b/compose-db.js
@@ -8,6 +8,8 @@ const ExitCodes = {
 };

 const PULLS_PER_PAGE = 100;
+const API_DELAY_MSEC = 2500;
+const API_MAX_RETRIES = 10;
 const API_RATE_LIMIT = `
  rateLimit {
    limit
@@ -59,7 +61,11 @@ class DataFetcher {
        });
    }

-    async fetchGithub(query) {
+    async delay(msec) {
+        return new Promise(resolve => setTimeout(resolve, msec));
+    }
+
+    async fetchGithub(query, retries = 0) {
        const init = {};
        init.method = "POST";
        init.headers = {};
@@ -68,13 +74,42 @@ class DataFetcher {
            init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`;
        } else if (process.env.GITHUB_TOKEN) {
            init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`;
+        } else {
+            console.error("    Unable to find environment variable: `GRAPHQL_TOKEN`. Did you forget to set it in your local environment or a root `.env` file?");
+            process.exitCode = buildCommon.ExitCodes.ParseFailure;
+            return [null, null];
        }

        init.body = JSON.stringify({
            query,
        });

-        return await fetch("https://api.github.com/graphql", init);
+        let res = await fetch("https://api.github.com/graphql", init);
+        let attempt = 0;
+
+        while (true) {
+            if (attempt > retries) {
+                return [res, null];
+            }
+
+            if (res.status === 200) {
+                try {
+                    const json = await res.json()
+                    return [res, json];
+                }
+                catch (err) {
+                    console.log(`    Failed due to invalid response body, retrying (${attempt}/${retries})...`);
+                }
+            }
+            else {
+                console.log(`    Failed with status ${res.status}, retrying (${attempt}/${retries})...`);
+            }
+
+            // GitHub API is flaky, so we add an extra delay to let it calm down a bit.
+            await this.delay(API_DELAY_MSEC);
+            attempt += 1;
+            res = await fetch("https://api.github.com/graphql", init);
+        }
    }

    async fetchGithubRest(query) {
@@ -86,6 +121,10 @@ class DataFetcher {
            init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`;
        } else if (process.env.GITHUB_TOKEN) {
            init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`;
+        } else {
+            console.error("    Unable to find environment variable: `GRAPHQL_TOKEN`. Did you forget to set it in your local environment or a root `.env` file?");
+            process.exitCode = buildCommon.ExitCodes.ParseFailure;
+            return null;
        }

        return await fetch(`${this.api_rest_path}${query}`, init);
@@ -99,14 +138,13 @@ class DataFetcher {
            }
            `;
    
-            const res = await this.fetchGithub(query);
-            if (res.status !== 200) {
+            const [res, data] = await this.fetchGithub(query);
+            if (res.status !== 200 || data == null) {
                this._handleResponseErrors(this.api_repository_id, res);
                process.exitCode = ExitCodes.RequestFailure;
                return;
            }
    
-            const data = await res.json();
            await this._logResponse(data, "_rate_limit");
            this._handleDataErrors(data);
    
@@ -191,14 +229,13 @@ class DataFetcher {
            }
            console.log(`    Requesting page ${page_text} of pull request data (${after_text}).`);
    
-            const res = await this.fetchGithub(query);
-            if (res.status !== 200) {
+            const [res, data] = await this.fetchGithub(query, API_MAX_RETRIES);
+            if (res.status !== 200 || data == null) {
                this._handleResponseErrors(this.api_repository_id, res);
                process.exitCode = ExitCodes.RequestFailure;
                return [];
            }
    
-            const data = await res.json();
            await this._logResponse(data, `data_page_${page}`);
            this._handleDataErrors(data);
    
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,7 +14,7 @@
        "dompurify": "^2.0.7",
        "lit-element": "^2.2.1",
        "marked": "^0.7.0",
-        "node-fetch": "^2.6.1",
+        "node-fetch": "^2.7.0",
        "posthtml": "^0.12.0",
        "rollup": "^1.24.0",
        "rollup-plugin-babel": "^4.3.3",
@@ -863,11 +863,23 @@
      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
    "node_modules/node-fetch": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
-      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "license": "MIT",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
      "engines": {
        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
      }
    },
    "node_modules/object-assign": {
@@ -1139,6 +1151,12 @@
        "node": ">=8.0"
      }
    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "license": "MIT"
+    },
    "node_modules/universalify": {
      "version": "0.1.2",
      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -1152,6 +1170,22 @@
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+      "license": "BSD-2-Clause"
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "license": "MIT",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
    "node_modules/wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
--- a/package.json
+++ b/package.json
@@ -15,7 +15,7 @@
    "dompurify": "^2.0.7",
    "lit-element": "^2.2.1",
    "marked": "^0.7.0",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "^2.7.0",
    "posthtml": "^0.12.0",
    "rollup": "^1.24.0",
    "rollup-plugin-babel": "^4.3.3",