From a5780b0894a2e20635fa9e789a6a000484583fe0 Mon Sep 17 00:00:00 2001 From: Yuri Sizov Date: Wed, 13 Jul 2022 20:43:54 +0300 Subject: [PATCH] Migrate to GraphQL API and add mergeability status --- README.md | 12 +- compose-db.js | 223 ++++++++++++++---- .../index/components/prs/PullRequestItem.js | 138 ++++++++++- .../index/components/prs/PullRequestList.js | 78 +++++- src/paths/index/components/teams/TeamItem.js | 2 +- src/paths/index/entry.js | 4 +- 6 files changed, 392 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index b68742a..ab9524e 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ Live website: https://godotengine.github.io/godot-team-reports/ ## Contributing -This project is written in JavaScript and built using Node.JS. HTML and CSS are used -for presentation. The end result of the build process is completely static and can -be server from any webserver, no Node.JS required. +This project is written in JavaScript and is built using Node.JS. HTML and CSS are +used for the presentation. The end result of the build process is completely static +and can be server from any web server, no Node.JS required. Front-end is designed in a reactive manner using industry standard Web Components (powered by `lit-element`). This provides native browser support, and results in a @@ -19,6 +19,12 @@ small overhead from the build process. To build the project locally you need to have Node.JS installed (12.x and newer should work just fine). +This project uses GitHub's GraphQL API. To fetch live data you need to generate +a [personal OAuth token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). +You can supply your token to the scripts using the `GITHUB_TOKEN` environment +variable. Note, that if you don't have member access to the organization, you +may not be able to access all the information used when generating the database. + 1. Clone or download the project. 2. From the project root run `npm install` or `yarn` to install dependencies. 3. Run `npm run build` or `yarn run build` to build the pages. diff --git a/compose-db.js b/compose-db.js index 5bfb013..0885ee0 100644 --- a/compose-db.js +++ b/compose-db.js @@ -6,9 +6,21 @@ const teams = {}; const reviewers = {}; const authors = {}; const pulls = []; -let page_count = 1; -const API_LINK_RE = /&page=([0-9]+)/g; +const PULLS_PER_PAGE = 100; +let page_count = 1; +let last_cursor = ""; + +const API_REPOSITORY_ID = `owner:"godotengine" name:"godot"`; +const API_RATE_LIMIT = ` + rateLimit { + limit + cost + remaining + resetAt + } +`; + // List of the keywords provided by https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue const GH_MAGIC_KEYWORDS = [ "close", "closes", "closed", @@ -18,57 +30,169 @@ const GH_MAGIC_KEYWORDS = [ const GH_MAGIC_RE = RegExp("(" + GH_MAGIC_KEYWORDS.join("|") + ") ([a-z0-9-_]+/[a-z0-9-_]+)?#([0-9]+)", "gi"); const GH_MAGIC_FULL_RE = RegExp("(" + GH_MAGIC_KEYWORDS.join("|") + ") https://github.com/([a-z0-9-_]+/[a-z0-9-_]+)/issues/([0-9]+)", "gi"); -async function fetchGithub(url) { +async function fetchGithub(query) { const init = {}; + init.method = "POST"; init.headers = {}; - init.headers["Accept"] = "application/vnd.github.v3+json"; + init.headers["Content-Type"] = "application/json"; + init.headers["Accept"] = "application/vnd.github.merge-info-preview+json"; if (process.env.GITHUB_TOKEN) { init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`; } - return await fetch(`https://api.github.com${url}`, init); + init.body = JSON.stringify({ + query, + }); + + return await fetch("https://api.github.com/graphql", init); +} + +function mapNodes(object) { + return object.edges.map((item) => item["node"]) } async function checkRates() { try { - const res = await fetchGithub("/rate_limit"); + const query = ` + query { + ${API_RATE_LIMIT} + } + `; + + const res = await fetchGithub(query); if (res.status !== 200) { - console.warn(" Failed to get the API rate limits."); + console.warn(` Failed to get the API rate limits; server responded with code ${res.status}`); return; } const data = await res.json(); - const core_apis = data.resources["core"]; - console.log(` Available API calls: ${core_apis.remaining}/${core_apis.limit}; resets at ${new Date(core_apis.reset * 1000).toISOString()}`); + const rate_limit = data.data["rateLimit"]; + console.log(` [$${rate_limit.cost}] Available API calls: ${rate_limit.remaining}/${rate_limit.limit}; resets at ${rate_limit.resetAt}`); } catch (err) { console.error(" Error checking the API rate limits: " + err); - return []; + return; } } async function fetchPulls(page) { try { + let after_cursor = ""; + if (last_cursor !== "") { + after_cursor = `after: "${last_cursor}"`; + } + + const query = ` + query { + ${API_RATE_LIMIT} + repository(${API_REPOSITORY_ID}) { + pullRequests(first:${PULLS_PER_PAGE} ${after_cursor} states: OPEN) { + totalCount + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + number + url + title + state + isDraft + mergeable + mergeStateStatus + createdAt + updatedAt + + bodyText + + baseRef { + name + } + + author { + login + avatarUrl + url + + ... on User { + id + } + } + + milestone { + id + title + url + } + + labels (first: 100) { + edges { + node { + id + name + color + } + } + } + + reviewRequests(first: 100) { + edges { + node { + id + requestedReviewer { + __typename + + ... on Team { + id + name + avatarUrl + slug + + parentTeam { + name + slug + } + } + + ... on User { + id + login + avatarUrl + } + } + } + } + } + } + } + } + } + } + `; + let page_text = page; if (page_count > 1) { page_text = `${page}/${page_count}`; } console.log(` Requesting page ${page_text} of pull request data.`); - const res = await fetchGithub(`/repos/godotengine/godot/pulls?state=open&per_page=100&page=${page}`); + const res = await fetchGithub(query); if (res.status !== 200) { + console.warn(` Failed to get pull requests for '${API_REPOSITORY_ID}'; server responded with code ${res.status}`); return []; } - const links = res.headers.get("link").split(","); - links.forEach((link) => { - if (link.includes('rel="last"')) { - const matches = API_LINK_RE.exec(link); - if (matches && matches[1]) { - page_count = Number(matches[1]); - } - } - }); + const data = await res.json(); + const rate_limit = data.data["rateLimit"]; + const repository = data.data["repository"]; + const pulls_data = mapNodes(repository.pullRequests); - return await res.json(); + console.log(` [$${rate_limit.cost}] Retrieved ${pulls_data.length} pull requests; processing...`); + + last_cursor = repository.pullRequests.pageInfo.endCursor; + page_count = Math.ceil(repository.pullRequests.totalCount / PULLS_PER_PAGE); + + return pulls_data; } catch (err) { console.error(" Error fetching pull request data: " + err); return []; @@ -76,25 +200,26 @@ async function fetchPulls(page) { } function processPulls(pullsRaw) { - console.log(" Processing retrieved pull requests."); pullsRaw.forEach((item) => { // Compile basic information about a PR. let pr = { "id": item.id, "public_id": item.number, - "url": item.html_url, - "diff_url": item.diff_url, - "patch_url": item.patch_url, + "url": item.url, + "diff_url": `${item.url}.diff`, + "patch_url": `${item.url}.patch`, "title": item.title, "state": item.state, - "is_draft": item.draft, + "is_draft": item.isDraft, "authored_by": null, - "created_at": item.created_at, - "updated_at": item.updated_at, + "created_at": item.createdAt, + "updated_at": item.updatedAt, - "target_branch": item.base.ref, + "target_branch": item.baseRef.name, + "mergeable_state": item.mergeable, + "mergeable_reason": item.mergeStateStatus, "labels": [], "milestone": null, "links": [], @@ -105,10 +230,10 @@ function processPulls(pullsRaw) { // Compose and link author information. const author = { - "id": item.user.id, - "user": item.user.login, - "avatar": item.user.avatar_url, - "url": item.user.html_url, + "id": item.author.id, + "user": item.author.login, + "avatar": item.author.avatarUrl, + "url": item.author.url, "pull_count": 0, }; pr.authored_by = author.id; @@ -124,12 +249,13 @@ function processPulls(pullsRaw) { pr.milestone = { "id": item.milestone.id, "title": item.milestone.title, - "url": item.milestone.html_url, + "url": item.milestone.url, }; } // Add labels, if available. - item.labels.forEach((labelItem) => { + let labels = mapNodes(item.labels); + labels.forEach((labelItem) => { pr.labels.push({ "id": labelItem.id, "name": labelItem.name, @@ -145,22 +271,26 @@ function processPulls(pullsRaw) { // Look for linked issues in the body. pr.links = extractLinkedIssues(item.body); + // Extract requested reviewers. + let review_requests = mapNodes(item.reviewRequests).map(it => it.requestedReviewer); + // Add teams, if available. - if (item.requested_teams.length > 0) { - item.requested_teams.forEach((teamItem) => { + let requested_teams = review_requests.filter(it => it["__typename"] === "Team"); + if (requested_teams.length > 0) { + requested_teams.forEach((teamItem) => { const team = { "id": teamItem.id, "name": teamItem.name, - "avatar": `https://avatars.githubusercontent.com/t/${teamItem.id}?s=40&v=4`, + "avatar": teamItem.avatarUrl, "slug": teamItem.slug, "full_name": teamItem.name, "full_slug": teamItem.slug, "pull_count": 0, }; // Include parent data into full name and slug. - if (teamItem.parent) { - team.full_name = `${teamItem.parent.name}/${team.name}`; - team.full_slug = `${teamItem.parent.slug}/${team.slug}`; + if (teamItem.parentTeam) { + team.full_name = `${teamItem.parentTeam.name}/${team.name}`; + team.full_slug = `${teamItem.parentTeam.slug}/${team.slug}`; } // Store the team if it hasn't been stored before. @@ -175,7 +305,7 @@ function processPulls(pullsRaw) { } else { // If there are no teams, use a fake "empty" team to track those PRs as well. const team = { - "id": -1, + "id": "", "name": "No team assigned", "avatar": "", "slug": "_", @@ -195,12 +325,13 @@ function processPulls(pullsRaw) { } // Add individual reviewers, if available - if (item.requested_reviewers.length > 0) { - item.requested_reviewers.forEach((reviewerItem) => { + let requested_reviewers = review_requests.filter(it => it["__typename"] === "User"); + if (requested_reviewers.length > 0) { + requested_reviewers.forEach((reviewerItem) => { const reviewer = { "id": reviewerItem.id, "name": reviewerItem.login, - "avatar": reviewerItem.avatar_url, + "avatar": reviewerItem.avatarUrl, "slug": reviewerItem.login, "pull_count": 0, }; @@ -275,7 +406,7 @@ async function main() { await checkRates(); console.log("[*] Fetching pull request data from GitHub."); - // Pages are starting with 1 (but 0 returns the same results). + // Pages are starting with 1 for better presentation. let page = 1; while (page <= page_count) { const pullsRaw = await fetchPulls(page); diff --git a/src/paths/index/components/prs/PullRequestItem.js b/src/paths/index/components/prs/PullRequestItem.js index 6de11a4..868a8ca 100644 --- a/src/paths/index/components/prs/PullRequestItem.js +++ b/src/paths/index/components/prs/PullRequestItem.js @@ -11,6 +11,11 @@ export default class PullRequestItem extends LitElement { --draft-background-color: #9db3c0; --stats-background-color: #f9fafa; + --mergeable-unknown-color: #939fa3; + --mergeable-no-color: #de1600; + --mergeable-maybe-color: #c49504; + --mergeable-yes-color: #50b923; + --stat-temp0-color: #000000; --stat-temp1-color: #383824; --stat-temp2-color: #645b2c; @@ -22,6 +27,7 @@ export default class PullRequestItem extends LitElement { --stat-temp8-color: #b31605; --stat-temp9-color: #d3001c; } + @media (prefers-color-scheme: dark) { :host { --pr-border-color: #0d1117; @@ -29,6 +35,11 @@ export default class PullRequestItem extends LitElement { --draft-background-color: #1e313c; --stats-background-color: #0f1316; + --mergeable-unknown-color: #4e5659; + --mergeable-no-color: #d31a3b; + --mergeable-maybe-color: #cbad1c; + --mergeable-yes-color: #3bc213; + --stat-temp0-color: #ffffff; --stat-temp1-color: #f0ed7e; --stat-temp2-color: #f5d94a; @@ -65,6 +76,7 @@ export default class PullRequestItem extends LitElement { } :host .pr-title-name { color: var(--g-font-color); + line-height: 24px; word-break: break-word; } @@ -97,18 +109,39 @@ export default class PullRequestItem extends LitElement { } :host .pr-label-dot { border-radius: 4px; + box-shadow: rgb(0 0 0 / 28%) 0 0 3px 0; display: inline-block; width: 8px; height: 8px; } :host .pr-label-name { - + padding-left: 3px; } :host .pr-milestone-value { font-weight: 700; } + :host .pr-mergeable-value, + :host .pr-mergeable-reason { + border-bottom: 1px dashed var(--g-font-color); + cursor: help; + font-weight: 700; + } + + :host .pr-mergeable-value--unknown { + color: var(--mergeable-unknown-color); + } + :host .pr-mergeable-value--no { + color: var(--mergeable-no-color); + } + :host .pr-mergeable-value--maybe { + color: var(--mergeable-maybe-color); + } + :host .pr-mergeable-value--yes { + color: var(--mergeable-yes-color); + } + :host .pr-time { } @@ -223,12 +256,14 @@ export default class PullRequestItem extends LitElement { `; } - @property({ type: Number }) id = -1; + @property({ type: String }) id = ''; @property({ type: String }) title = ''; @property({ type: String, reflect: true }) url = ''; @property({ type: String, reflect: true }) diff_url = ''; @property({ type: String, reflect: true }) patch_url = ''; @property({ type: Boolean }) draft = false; + @property({ type: String }) mergeable_state = ''; + @property({ type: String }) mergeable_reason = ''; @property({ type: Array }) labels = []; @property({ type: String, reflect: true }) milestone = ''; @property({ type: String, reflect: true }) branch = ''; @@ -238,6 +273,8 @@ export default class PullRequestItem extends LitElement { @property({ type: Object }) author = null; @property({ type: Array }) teams = []; + + getStatTemp(value, factor) { let temp = Math.floor(value / factor); if (temp > 9) { @@ -247,6 +284,76 @@ export default class PullRequestItem extends LitElement { return temp; } + getMergeableStateText(value, reason) { + const descriptions = { + 'UNKNOWN': "unknown", + + 'CONFLICTING': "no", + 'MERGEABLE': "yes", + }; + + if (typeof descriptions[value] === "undefined") { + return value.toLowerCase; + } + + if (value === 'MERGEABLE' && ![ 'CLEAN', 'HAS_HOOKS', 'UNSTABLE' ].includes(reason)) { + return "maybe"; + } + return descriptions[value]; + } + + getMergeableStateDescription(value) { + const descriptions = { + 'UNKNOWN': "The mergeability of the pull request is not calculated.", + + 'CONFLICTING': "The pull request cannot be merged due to merge conflicts.", + 'MERGEABLE': "The pull request can be merged.", + }; + + if (typeof descriptions[value] === "undefined") { + return value; + } + return descriptions[value]; + } + + getMergeableReasonText(value) { + const descriptions = { + 'UNKNOWN': "unknown", + + 'BEHIND': "outdated branch", + 'BLOCKED': "blocked", + 'CLEAN': "clean", + 'DIRTY': "merge conflicts", + 'DRAFT': "draft", + 'HAS_HOOKS': "passing status w/ hooks", + 'UNSTABLE': "non-passing status", + }; + + if (typeof descriptions[value] === "undefined") { + return value; + } + return descriptions[value]; + } + + getMergeableReasonDescription(value) { + const descriptions = { + 'UNKNOWN': "The state cannot currently be determined.", + + 'BEHIND': "The head ref is out of date.", + 'BLOCKED': "The merge is blocked. It likely requires an approving review.", + 'CLEAN': "Mergeable and passing commit status.", + 'DIRTY': "The merge commit cannot be cleanly created.", + 'DRAFT': "The merge is blocked due to the pull request being a draft.", + 'HAS_HOOKS': "Mergeable with passing commit status and pre-receive hooks.", + 'UNSTABLE': "Mergeable with non-passing commit status.", + }; + + if (typeof descriptions[value] === "undefined") { + return value; + } + return descriptions[value]; + } + render(){ const created_days = greports.format.getDaysSince(this.created_at); const stale_days = greports.format.getDaysSince(this.updated_at); @@ -266,6 +373,12 @@ export default class PullRequestItem extends LitElement { authorClassList.push("pr-author-value--hot"); } + // Keep it to two columns, but if there isn't enough labels, keep it to one. + let labels_height = Math.ceil(this.labels.length / 2) * 20; + if (labels_height < 60) { + labels_height = 60; + } + return html`
-
+
${this.labels.map((item) => { return html`
+
+ mergeable: + + ${this.getMergeableStateText(this.mergeable_state, this.mergeable_reason)} + + ${(this.mergeable_reason !== 'UNKNOWN') ? html` + | + + ${this.getMergeableReasonText(this.mergeable_reason)} + + ` : html`` + } +
diff --git a/src/paths/index/components/prs/PullRequestList.js b/src/paths/index/components/prs/PullRequestList.js index a29c524..c97e619 100644 --- a/src/paths/index/components/prs/PullRequestList.js +++ b/src/paths/index/components/prs/PullRequestList.js @@ -10,11 +10,19 @@ export default class PullRequestList extends LitElement { :host { --pulls-background-color: #e5edf8; --pulls-toolbar-color: #9bbaed; + + --sort-color: #5c7bb6; + --sort-color-hover: #2862cd; + --sort-color-active: #2054b5; } @media (prefers-color-scheme: dark) { :host { --pulls-background-color: #191d23; --pulls-toolbar-color: #222c3d; + + --sort-color: #4970ad; + --sort-color-hover: #5b87de; + --sort-color-active: #6b9aea; } } @@ -95,16 +103,16 @@ export default class PullRequestList extends LitElement { } :host .pulls-sort-action { - color: var(--link-font-color); + color: var(--sort-color); cursor: pointer; } :host .pulls-sort-action:hover { - color: var(--link-font-color-hover); + color: var(--sort-color-hover); } :host .pulls-sort-action--active, :host .pulls-sort-action--active:hover { - color: var(--link-font-color-inactive); + color: var(--sort-color-active); cursor: default; text-decoration: underline; } @@ -142,7 +150,7 @@ export default class PullRequestList extends LitElement { @property({ type: Array }) pulls = []; @property({ type: Object }) teams = {}; - @property( { type: Number }) selected_group = -1; + @property( { type: String }) selected_group = ""; @property({ type: Boolean }) selected_is_person = false; @property({ type: Object }) authors = {}; @@ -153,6 +161,7 @@ export default class PullRequestList extends LitElement { this._sortDirection = "desc"; this._showDraft = false; this._filterMilestone = "4.0"; + this._filterMergeable = ""; } onSortClicked(sortField, event) { @@ -175,6 +184,29 @@ export default class PullRequestList extends LitElement { this.requestUpdate(); } + onMergeableChanged(event) { + this._filterMergeable = event.target.value; + this.requestUpdate(); + } + + getMergeableFilterValue(state, reason) { + const descriptions = { + 'UNKNOWN': "unknown", + + 'CONFLICTING': "no", + 'MERGEABLE': "yes", + }; + + if (typeof descriptions[state] === "undefined") { + return "unknown"; + } + + if (state === 'MERGEABLE' && ![ 'CLEAN', 'HAS_HOOKS', 'UNSTABLE' ].includes(reason)) { + return "maybe"; + } + return descriptions[state]; + } + render(){ const milestones = []; @@ -204,6 +236,10 @@ export default class PullRequestList extends LitElement { this._filterMilestone = ""; } + let mergeables = [ + "no", "maybe", "yes" + ]; + pulls = pulls.filter((item) => { if (!this._showDraft && item.is_draft) { return false; @@ -211,6 +247,9 @@ export default class PullRequestList extends LitElement { if (this._filterMilestone !== "" && item.milestone && item.milestone.title !== this._filterMilestone) { return false; } + if (this._filterMergeable !== "" && this.getMergeableFilterValue(item.mergeable_state, item.mergeable_reason) !== this._filterMergeable) { + return false; + } return true; }); @@ -234,9 +273,9 @@ export default class PullRequestList extends LitElement { @@ -247,8 +286,25 @@ export default class PullRequestList extends LitElement { ${milestones.map((item) => { return html` + ` + })} + + + + + mergeable: +