diff --git a/compose-db.js b/compose-db.js index a45d69b..2bae0e5 100644 --- a/compose-db.js +++ b/compose-db.js @@ -38,8 +38,8 @@ class DataFetcher { } } - _handleResponseErrors(res) { - console.warn(` Failed to get pull requests for '${API_REPOSITORY_ID}'; server responded with ${res.status} ${res.statusText}`); + _handleResponseErrors(queryID, res) { + console.warn(` Failed to get data from '${queryID}'; server responded with ${res.status} ${res.statusText}`); const retry_header = res.headers.get("Retry-After"); if (retry_header) { console.log(` Retry after: ${retry_header}`); @@ -99,7 +99,7 @@ class DataFetcher { const res = await this.fetchGithub(query); if (res.status !== 200) { - this._handleResponseErrors(res); + this._handleResponseErrors(API_REPOSITORY_ID, res); process.exitCode = ExitCodes.RequestFailure; return; } @@ -201,7 +201,7 @@ class DataFetcher { const res = await this.fetchGithub(query); if (res.status !== 200) { - this._handleResponseErrors(res); + this._handleResponseErrors(API_REPOSITORY_ID, res); process.exitCode = ExitCodes.RequestFailure; return []; } @@ -233,7 +233,7 @@ class DataFetcher { const res = await this.fetchGithubRest(query); if (res.status !== 200) { - this._handleResponseErrors(res); + this._handleResponseErrors(query, res); process.exitCode = ExitCodes.RequestFailure; return []; } @@ -259,7 +259,10 @@ class DataProcessor { constructor() { this.authors = {}; this.pulls = []; - this.files = []; + this.branches = []; + this.files = {}; + + this._pullsByFile = {}; } _explainFileType(type) { @@ -299,6 +302,11 @@ class DataProcessor { "files": [], }; + // Store the target branch if it hasn't been stored. + if (!this.branches.includes(pr.target_branch)) { + this.branches.push(pr.target_branch); + } + // Compose and link author information. const author = { "id": "", @@ -360,6 +368,17 @@ class DataProcessor { }); this.pulls.push(pr); + + // Cache the pull information for every file that it includes. + if (typeof this._pullsByFile[pr.target_branch] === "undefined") { + this._pullsByFile[pr.target_branch] = {}; + } + pr.files.forEach((file) => { + if (typeof this._pullsByFile[pr.target_branch][file.path] === "undefined") { + this._pullsByFile[pr.target_branch][file.path] = []; + } + this._pullsByFile[pr.target_branch][file.path].push(pr.public_id); + }) }); } catch (err) { console.error(" Error parsing pull request data: " + err); @@ -367,23 +386,42 @@ class DataProcessor { } } - processFiles(filesRaw) { + processFiles(targetBranch, filesRaw) { try { + this.files[targetBranch] = []; + filesRaw.forEach((item) => { let file = { "type": this._explainFileType(item.type), "name": item.path.split("/").pop(), "path": item.path, "parent": "", + "pulls": [], }; + // Store the parent path for future reference. let parentPath = item.path.split("/"); parentPath.pop(); if (parentPath.length > 0) { file.parent = parentPath.join("/"); } - this.files.push(file); + // Fetch the PRs touching this file or files in this folder from the cache. + if (typeof this._pullsByFile[targetBranch] !== "undefined") { + for (let filePath in this._pullsByFile[targetBranch]) { + if (filePath !== file.path && filePath.indexOf(file.path + "/") < 0) { + continue; + } + + this._pullsByFile[targetBranch][filePath].forEach((pullNumber) => { + if (!file.pulls.includes(pullNumber)) { + file.pulls.push(pullNumber); + } + }); + } + } + + this.files[targetBranch].push(file); }); } catch (err) { console.error(" Error parsing repository file system: " + err); @@ -415,24 +453,26 @@ async function main() { await dataFetcher.checkRates(); checkForExit(); - // console.log("[*] Fetching pull request data from GitHub."); - // // Pages are starting with 1 for better presentation. - // let page = 1; - // while (page <= page_count) { - // const pullsRaw = await dataFetcher.fetchPulls(page); - // dataProcessor.processPulls(pullsRaw); - // checkForExit(); - // page++; + console.log("[*] Fetching pull request data from GitHub."); + // Pages are starting with 1 for better presentation. + let page = 1; + while (page <= page_count) { + const pullsRaw = await dataFetcher.fetchPulls(page); + dataProcessor.processPulls(pullsRaw); + checkForExit(); + page++; - // // Wait for a bit before proceeding to avoid hitting the secondary rate limit in GitHub API. - // // See https://docs.github.com/en/rest/guides/best-practices-for-integrators#dealing-with-secondary-rate-limits. - // await delay(1500); - // } + // Wait for a bit before proceeding to avoid hitting the secondary rate limit in GitHub API. + // See https://docs.github.com/en/rest/guides/best-practices-for-integrators#dealing-with-secondary-rate-limits. + await delay(1500); + } console.log("[*] Fetching repository file system from GitHub."); - const filesRaw = await dataFetcher.fetchFiles("master"); - dataProcessor.processFiles(filesRaw); - checkForExit(); + for (let branch of dataProcessor.branches) { + const filesRaw = await dataFetcher.fetchFiles(branch); + dataProcessor.processFiles(branch, filesRaw); + checkForExit(); + } console.log("[*] Checking the rate limits after.") await dataFetcher.checkRates(); @@ -443,6 +483,7 @@ async function main() { "generated_at": Date.now(), "authors": dataProcessor.authors, "pulls": dataProcessor.pulls, + "branches": dataProcessor.branches, "files": dataProcessor.files, }; try { diff --git a/src/paths/index/components/files/FileList.js b/src/paths/index/components/files/FileList.js index 291afd0..5e9e507 100644 --- a/src/paths/index/components/files/FileList.js +++ b/src/paths/index/components/files/FileList.js @@ -46,8 +46,12 @@ export default class FileList extends LitElement { `; } + @property({ type: Array }) branches = []; @property({ type: Object }) files = {}; - @property({ type: Array }) selected = []; + + @property({ type: String }) selectedRepository = "godotengine/godot"; + @property({ type: String }) selectedBranch = "master"; + @property({ type: Array }) selectedFolders = []; constructor() { super(); @@ -58,34 +62,34 @@ export default class FileList extends LitElement { return; } - const entryIndex = this.selected.indexOf(entryPath); + const entryIndex = this.selectedFolders.indexOf(entryPath); if (entryIndex >= 0) { - this.selected.splice(entryIndex, 1); + this.selectedFolders.splice(entryIndex, 1); } else { - this.selected.push(entryPath); + this.selectedFolders.push(entryPath); } this.requestUpdate(); } - renderFolder(levelEntries) { + renderFolder(branchFiles, folderFiles) { return html`
- ${(levelEntries.length > 0) ? - levelEntries.map((item) => { + ${(folderFiles.length > 0) ? + folderFiles.map((item) => { return html`
- ${(this.selected.includes(item.path)) ? - this.renderFolder(this.files[item.path] || []) : null + ${(this.selectedFolders.includes(item.path)) ? + this.renderFolder(branchFiles, branchFiles[item.path] || []) : null }
`; @@ -98,16 +102,17 @@ export default class FileList extends LitElement { } render() { - const topLevel = this.files[""] || []; + const branchFiles = this.files[this.selectedBranch]; + const topLevel = branchFiles[""] || []; return html`
- ${this.renderFolder(topLevel)} + ${this.renderFolder(branchFiles, topLevel)}
`; } diff --git a/src/paths/index/entry.js b/src/paths/index/entry.js index 2df4376..41f78d0 100644 --- a/src/paths/index/entry.js +++ b/src/paths/index/entry.js @@ -35,6 +35,7 @@ export default class EntryComponent extends LitElement { this._isLoading = true; this._generatedAt = null; + this._branches = []; this._files = {}; this._requestData(); @@ -56,37 +57,49 @@ export default class EntryComponent extends LitElement { if (data) { this._generatedAt = data.generated_at; - data.files.forEach((file) => { - if (file.type === "file" || file.type === "folder") { - if (typeof this._files[file.parent] === "undefined") { - this._files[file.parent] = []; - } - - this._files[file.parent].push(file); + data.branches.forEach((branch) => { + if (typeof data.files[branch] === "undefined") { + return; } - }); - for (let folderName in this._files) { - this._files[folderName].sort((a, b) => { - if (a.type === "folder" && b.type !== "folder") { - return -1; - } - if (b.type === "folder" && a.type !== "folder") { - return 1; - } + this._branches.push(branch); + const branchFiles = {}; + + data.files[branch].forEach((file) => { + if (file.type === "file" || file.type === "folder") { + if (typeof branchFiles[file.parent] === "undefined") { + branchFiles[file.parent] = []; + } - const a_name = a.path.toLowerCase(); - const b_name = b.path.toLowerCase(); - - if (a_name > b_name) return 1; - if (a_name < b_name) return -1; - return 0; + branchFiles[file.parent].push(file); + } }); - } + + for (let folderName in branchFiles) { + branchFiles[folderName].sort((a, b) => { + if (a.type === "folder" && b.type !== "folder") { + return -1; + } + if (b.type === "folder" && a.type !== "folder") { + return 1; + } + + const a_name = a.path.toLowerCase(); + const b_name = b.path.toLowerCase(); + + if (a_name > b_name) return 1; + if (a_name < b_name) return -1; + return 0; + }); + } + + this._files[branch] = branchFiles; + }); } else { this._generatedAt = null; - this._files = []; + this._branches = []; + this._files = {}; } this._isLoading = false; @@ -104,6 +117,7 @@ export default class EntryComponent extends LitElement { ` : html`