diff --git a/compose.js b/compose.js index dc9c912..fb041f1 100644 --- a/compose.js +++ b/compose.js @@ -1,22 +1,44 @@ const fs = require('fs').promises; const path = require('path'); +const fetch = require('node-fetch'); -async function fetchPulls() { +const teams = {}; +const pulls = []; +let page_count = 1; + +const LINK_RE = /&page=([0-9]+)/g; + +async function fetchPulls(page) { try { - const json = await fs.readFile("pulls.raw.json", {encoding: "utf-8", flag: "r"}); - return JSON.parse(json); + let page_text = page; + if (page_count > 1) { + page_text = `${page}/${page_count}`; + } + console.log(` Requesting page ${page_text} of pull request data.`); + const res = await fetch(`https://api.github.com/repos/godotengine/godot/pulls?state=open&per_page=100&page=${page}`); + if (res.status !== 200) { + return []; + } + + const links = res.headers.get("link").split(","); + links.forEach((link) => { + if (link.includes('rel="last"')) { + const matches = LINK_RE.exec(link); + if (matches && matches[1]) { + page_count = Number(matches[1]); + } + } + }); + + return await res.json(); } catch (err) { - console.error("Error fetching the pull requests: " + err); - return {}; + console.error("Error fetching pull request data: " + err); + return []; } } -async function processPulls() { - const pullsRaw = await fetchPulls(); - - let teams = {}; - let pulls = []; - +function processPulls(pullsRaw) { + console.log(" Processing retrieved pull requests."); pullsRaw.forEach((item) => { // Compile basic information about a PR. let pr = { @@ -59,15 +81,15 @@ async function processPulls() { // Add labels, if available. item.labels.forEach((labelItem) => { pr.labels.push({ - "id": labelItem.id, - "name": labelItem.name, - "color": "#" + labelItem.color + "id": labelItem.id, + "name": labelItem.name, + "color": "#" + labelItem.color }); }); pr.labels.sort((a, b) => { - if (a.name > b.name) return 1; - if (a.name < b.name) return -1; - return 0; + if (a.name > b.name) return 1; + if (a.name < b.name) return -1; + return 0; }); // Add teams, if available. @@ -97,13 +119,32 @@ async function processPulls() { pulls.push(pr); }); +} +async function main() { + console.log("[*] Building local pull request database."); + + console.log("[*] Fetching pull request data from GitHub."); + // Pages are starting with 1 (but 0 returns the same results). + let page = 1; + while (page <= page_count) { + const pullsRaw = await fetchPulls(page); + processPulls(pullsRaw); + page++; + } + + console.log("[*] Finalizing database.") const output = { "generated_at": Date.now(), "teams": teams, "pulls": pulls, }; - fs.writeFile("out/data.json", JSON.stringify(output), { encoding: "utf-8" }); + try { + console.log("[*] Storing database to file.") + await fs.writeFile("out/data.json", JSON.stringify(output), {encoding: "utf-8"}); + } catch (err) { + console.error("Error saving database file: " + err); + } } -processPulls(); +main(); diff --git a/package-lock.json b/package-lock.json index 5999e23..355d454 100644 --- a/package-lock.json +++ b/package-lock.json @@ -716,6 +716,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", diff --git a/package.json b/package.json index 6a0226a..5496a56 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dompurify": "^2.0.7", "lit-element": "^2.2.1", "marked": "^0.7.0", + "node-fetch": "^2.6.1", "posthtml": "^0.12.0", "rollup": "^1.24.0", "rollup-plugin-babel": "^4.3.3", diff --git a/src/paths/index/entry.js b/src/paths/index/entry.js index 98d3ed6..bb79270 100644 --- a/src/paths/index/entry.js +++ b/src/paths/index/entry.js @@ -208,7 +208,7 @@ export default class EntryComponent extends LitElement { title="Show older PRs first" @click="${this.onSortClicked.bind(this, "age")}" > - Age + Lifetime |
-
+
lifetime:
-
+
stale for: