diff --git a/build/res/redirect_index.html b/build/res/redirect_index.html new file mode 100644 index 0000000..e950cf5 --- /dev/null +++ b/build/res/redirect_index.html @@ -0,0 +1,11 @@ + + + + Redirecting… + + + + +

Redirecting…

+ Click here if you are not redirected. + diff --git a/compose-db.js b/compose-db.js index 1274d83..caee2c0 100644 --- a/compose-db.js +++ b/compose-db.js @@ -263,33 +263,69 @@ class DataProcessor { this.artifacts = {}; } + readExistingData(existingData) { + if (typeof existingData.commits !== "undefined") { + this.commits = existingData.commits; + } + if (typeof existingData.checks !== "undefined") { + this.checks = existingData.checks; + } + if (typeof existingData.runs !== "undefined") { + this.runs = existingData.runs; + } + if (typeof existingData.artifacts !== "undefined") { + this.artifacts = existingData.artifacts; + } + } + processRuns(runsRaw) { try { + // We will be adding items to the front, so reversing is + // necessary. + runsRaw.reverse(); + runsRaw.forEach((item) => { - // Compile basic information about a commit. - let commit = { - "hash": item.oid, - "title": item.messageHeadline, - "committed_date": item.committedDate, - "checks": [], - }; + // Check if this commit is already tracked. + let commit = this.commits.find((it) => { + return it.hash === item.oid; + }); + + if (!commit) { + // Compile basic information about a commit. + commit = { + "hash": item.oid, + "title": item.messageHeadline, + "committed_date": item.committedDate, + "checks": [], + }; + this.commits.unshift(commit); + } const checkSuites = mapNodes(item.checkSuites); checkSuites.forEach((checkItem) => { - // Compile basic information about a check suite. - let check = { - "check_id": checkItem.databaseId, - "check_url": checkItem.url, - "status": checkItem.status, - "conclusion": checkItem.conclusion, + let check = this.checks[checkItem.databaseId]; - "created_at": checkItem.createdAt, - "updated_at": checkItem.updatedAt, + if (typeof check === "undefined") { + // Compile basic information about a check suite. + check = { + "check_id": checkItem.databaseId, + "check_url": checkItem.url, + "status": checkItem.status, + "conclusion": checkItem.conclusion, - "workflow": null, - }; + "created_at": checkItem.createdAt, + "updated_at": checkItem.updatedAt, - if (checkItem.workflowRun) { + "workflow": "", + }; + this.checks[check.check_id] = check; + } else { + check.status = checkItem.status; + check.conclusion = checkItem.conclusion; + check.updatedAt = checkItem.updatedAt; + } + + if (check.workflow === "" && checkItem.workflowRun) { const runItem = checkItem.workflowRun; let run = { "name": runItem.workflow.name, @@ -303,11 +339,13 @@ class DataProcessor { check.workflow = run.run_id; } - this.checks[check.check_id] = check; - commit.checks.push(check.check_id); - }); - this.commits.push(commit); + // Existing data may contain this commit, but not all of + // its checks. + if (commit.checks.indexOf(check.check_id) < 0) { + commit.checks.push(check.check_id); + } + }); }); } catch (err) { console.error(" Error parsing pull request data: " + err); @@ -315,6 +353,21 @@ class DataProcessor { } } + getIncompleteRuns() { + let runs = []; + + for (let runId in this.runs) { + const runData = this.runs[runId]; + if (runData.artifacts.length > 0) { + continue; + } + + runs.push(runId); + } + + return runs; + } + processArtifacts(runId, artifactsRaw) { try { artifactsRaw.forEach((item) => { @@ -335,6 +388,37 @@ class DataProcessor { process.exitCode = ExitCodes.ParseFailure; } } + + getLatestArtifacts() { + let latest = {}; + + this.commits.forEach((commit) => { + for (let checkId of commit.checks) { + const check = this.checks[checkId]; + if (check.workflow === "") { + continue; + } + + const run = this.runs[check.workflow]; + run.artifacts.forEach((artifact) => { + if (typeof latest[artifact.name] !== "undefined") { + return; // Continue; + } + + latest[artifact.name] = { + "commit_hash": commit.hash, + "check_id": check.check_id, + "workflow_name": run.name, + "artifact_id": artifact.id, + "artifact_name": artifact.name, + "artifact_size": artifact.size, + }; + }); + } + }); + + return latest; + } } class DataIO { @@ -369,13 +453,13 @@ class DataIO { try { console.log("[*] Loading existing database from a file."); - // const dataPath = `./out/data/${this.data_owner}.${this.data_repo}.${this.data_branch}.json`; - // await fs.access(dataPath, fsConstants.R_OK); - // const existingData = await fs.readFile(dataPath); + const dataPath = `./out/data/${this.data_owner}.${this.data_repo}.${this.data_branch}.json`; + await fs.access(dataPath, fsConstants.R_OK); + const fileRaw = await fs.readFile(dataPath, {encoding: "utf-8"}); + + return JSON.parse(fileRaw); } catch (err) { - console.error(" Error loading existing database file: " + err); - process.exitCode = ExitCodes.IOFailure; - return; + return {}; } } @@ -392,6 +476,48 @@ class DataIO { return; } } + + async createRedirects(artifacts) { + let redirectTemplate = ""; + + try { + const dataPath = `./build/res/redirect_index.html`; + await fs.access(dataPath, fsConstants.R_OK); + redirectTemplate = await fs.readFile(dataPath, {encoding: "utf-8"}); + + if (redirectTemplate === "") { + throw new Error("File is missing."); + } + } catch (err) { + console.error(" Error loading a redirect template: " + err); + process.exitCode = ExitCodes.IOFailure; + return; + } + + await ensureDir("./out"); + await ensureDir("./out/download"); + await ensureDir(`./out/download/${this.data_owner}`); + await ensureDir(`./out/download/${this.data_owner}/${this.data_repo}`); + await ensureDir(`./out/download/${this.data_owner}/${this.data_repo}/${this.data_branch}`); + + const outputDir = `./out/download/${this.data_owner}/${this.data_repo}/${this.data_branch}`; + for (let artifactName in artifacts) { + await ensureDir(`${outputDir}/${artifactName}`); + + try { + const artifact = artifacts[artifactName]; + const artifactPath = `https://github.com/godotengine/godot/suites/${artifact.check_id}/artifacts/${artifact.artifact_id}`; + + const redirectPage = redirectTemplate.replace(/\{\{REDIRECT_PATH\}\}/g, artifactPath); + await fs.writeFile(`${outputDir}/${artifactName}/index.html`, redirectPage, {encoding: "utf-8"}); + console.log(` Created a redirect at ${outputDir}/${artifactName}.`) + } catch (err) { + console.error(` Error saving a redirect page for "${artifactName}": ` + err); + process.exitCode = ExitCodes.IOFailure; + return; + } + } + } } function mapNodes(object) { @@ -455,14 +581,14 @@ async function main() { dataIO.parseArgs(); checkForExit(); - // await dataIO.loadConfig(); - // checkForExit(); - console.log(`[*] Configured for the "${dataIO.data_owner}/${dataIO.data_repo}" repository; branch ${dataIO.data_branch}.`); const dataFetcher = new DataFetcher(dataIO.data_owner, dataIO.data_repo); const dataProcessor = new DataProcessor(); + const existingData = await dataIO.loadData(); + dataProcessor.readExistingData(existingData); + console.log("[*] Checking the rate limits before."); await dataFetcher.checkRates(); checkForExit(); @@ -474,7 +600,7 @@ async function main() { checkForExit(); console.log("[*] Fetching artifact data from GitHub."); - for (let runId in dataProcessor.runs) { + for (let runId of dataProcessor.getIncompleteRuns()) { const artifactsRaw = await dataFetcher.fetchArtifacts(runId); checkForExit(); dataProcessor.processArtifacts(runId, artifactsRaw); @@ -489,6 +615,8 @@ async function main() { await dataFetcher.checkRates(); checkForExit(); + const latestArtifacts = dataProcessor.getLatestArtifacts(); + console.log("[*] Finalizing database.") const output = { "generated_at": Date.now(), @@ -496,11 +624,16 @@ async function main() { "checks": dataProcessor.checks, "runs": dataProcessor.runs, "artifacts": dataProcessor.artifacts, + "latest": latestArtifacts, }; await dataIO.saveData(output, `${dataIO.data_owner}.${dataIO.data_repo}.${dataIO.data_branch}.json`); checkForExit(); + console.log("[*] Creating stable download paths."); + await dataIO.createRedirects(latestArtifacts); + checkForExit(); + console.log("[*] Database built."); } diff --git a/src/paths/index/components/commits/CommitItem.js b/src/paths/index/components/commits/CommitItem.js index c1139af..f92abd3 100644 --- a/src/paths/index/components/commits/CommitItem.js +++ b/src/paths/index/components/commits/CommitItem.js @@ -54,9 +54,11 @@ export default class CommitItem extends LitElement { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; - border-bottom: 2px solid var(--g-background-extra-color); padding: 12px 10px; } + :host .workflow + .workflow { + border-top: 2px solid var(--g-background-extra-color); + } :host .workflow-artifacts { display: flex; @@ -96,6 +98,13 @@ export default class CommitItem extends LitElement { @property({ type: String }) repository = ''; render(){ + const [...workflows] = this.workflows; + workflows.sort((a,b) => { + if (a.name_sanitized > b.name_sanitized) return 1; + if (a.name_sanitized < b.name_sanitized) return -1; + return 0; + }); + return html`
@@ -110,7 +119,7 @@ export default class CommitItem extends LitElement {
${this.title}
- ${this.workflows.map((item) => { + ${workflows.map((item) => { return html`
${item.name}
diff --git a/src/paths/index/components/commits/CommitList.js b/src/paths/index/components/commits/CommitList.js index 6bbe140..17e7f42 100644 --- a/src/paths/index/components/commits/CommitList.js +++ b/src/paths/index/components/commits/CommitList.js @@ -1,6 +1,7 @@ import { LitElement, html, css, customElement, property } from 'lit-element'; import CommitItem from "./CommitItem"; +import LatestItem from "./LatestItem"; @customElement('gr-commit-list') export default class CommitList extends LitElement { @@ -57,11 +58,60 @@ export default class CommitList extends LitElement { @property({ type: Object }) checks = {}; @property({ type: Object }) runs = {}; @property({ type: Object }) artifacts = {}; + @property({ type: Object }) latest = {}; @property({ type: String }) selectedRepository = ""; @property({ type: String }) selectedBranch = ""; @property({ type: Boolean, reflect: true }) loading = false; + constructor() { + super(); + + this._workflowsPerCommit = {}; + } + + _updateWorkflows() { + this._workflowsPerCommit = {}; + + this.commits.forEach((item) => { + let workflows = []; + + for (let checkId in this.checks) { + const check = this.checks[checkId]; + if (item.checks.indexOf(check.check_id) < 0) { + continue; + } + + if (check.workflow === "" || typeof this.runs[check.workflow] === "undefined") { + continue; + } + + const run = this.runs[check.workflow]; + if (run.artifacts.length === 0) { + continue; + } + + workflows.push({ + "name": run.name, + "name_sanitized": run.name.replace(/([^a-zA-Z0-9_\- ]+)/g, "").trim().toLowerCase(), + "check_id": check.check_id, + "artifacts": run.artifacts, + }); + } + + this._workflowsPerCommit[item.hash] = workflows; + }); + } + + update(changedProperties) { + // Only recalculate when class properties change; skip for manual updates. + if (changedProperties.size > 0) { + this._updateWorkflows(); + } + + super.update(changedProperties); + } + render(){ if (this.selectedBranch === "") { return html``; @@ -69,42 +119,19 @@ export default class CommitList extends LitElement { if (this.loading) { return html` Loading artifacts... - ` + `; } return html`
+ + ${this.commits.map((item) => { - let workflows = []; - - for (let checkId in this.checks) { - const check = this.checks[checkId]; - if (item.checks.indexOf(check.check_id) < 0) { - continue; - } - - if (check.workflow == null || typeof this.runs[check.workflow] === "undefined") { - continue; - } - - const run = this.runs[check.workflow]; - if (run.artifacts.length === 0) { - continue; - } - - workflows.push({ - "name": run.name, - "name_sanitized": run.name.replace(/([^a-zA-Z0-9_\- ]+)/g, "").trim().toLowerCase(), - "check_id": check.check_id, - "artifacts": run.artifacts, - }); - } - - workflows.sort((a,b) => { - if (a.name_sanitized > b.name_sanitized) return 1; - if (a.name_sanitized < b.name_sanitized) return -1; - return 0; - }); + const workflows = this._workflowsPerCommit[item.hash]; return html` { + if (a.name_sanitized > b.name_sanitized) return 1; + if (a.name_sanitized < b.name_sanitized) return -1; + return 0; + }); + } + + update(changedProperties) { + // Only recalculate when class properties change; skip for manual updates. + if (changedProperties.size > 0) { + this._updateWorkflows(); + } + + super.update(changedProperties); + } + + render(){ + return html` +
+
+ Latest +
+
Builds may be from different runs, depending on their availability.
+
+ ${this._latestByWorkflow.map((item) => { + return html` +
+
${item.name}
+
+ ${item.artifacts.map((artifact) => { + return html` + + + ${artifact.artifact_name} + + (${greports.format.humanizeBytes(artifact.artifact_size)}) + + `; + })} +
+
+ `; + })} +
+
+ `; + } +} diff --git a/src/paths/index/entry.js b/src/paths/index/entry.js index 2e60704..e1a0dee 100644 --- a/src/paths/index/entry.js +++ b/src/paths/index/entry.js @@ -120,6 +120,7 @@ export default class EntryComponent extends LitElement { let checks = {}; let runs = {}; let artifacts = {}; + let latest = {}; if (this._selectedBranch !== "" && typeof this._branchData[this._selectedBranch] !== "undefined") { const branchData = this._branchData[this._selectedBranch]; @@ -128,6 +129,7 @@ export default class EntryComponent extends LitElement { checks = branchData.checks; runs = branchData.runs; artifacts = branchData.artifacts; + latest = branchData.latest; } return html` @@ -153,6 +155,7 @@ export default class EntryComponent extends LitElement { .checks="${checks}" .runs="${runs}" .artifacts="${artifacts}" + .latest="${latest}" .selectedRepository="${this._selectedRepository}" .selectedBranch="${this._selectedBranch}"