godotengine/godot-commit-artifacts
1
0
Fork 0
mirror of https://github.com/godotengine/godot-commit-artifacts.git synced 2025-12-31 05:48:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Reuse existing data and display evergreen artifacts

Browse Source
This commit is contained in:
Yuri Sizov
2023-03-28 18:31:46 +02:00
parent 89102f1687
commit a896d2d08e
6 changed files with 422 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
build/res/redirect_index.html Normal file
View File

@@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en-US">
<meta charset="utf-8">
<title>Redirecting&hellip;</title>
<link rel="canonical" href="{{REDIRECT_PATH}}">
<script>location="{{REDIRECT_PATH}}"</script>
<meta http-equiv="refresh" content="0; url={{REDIRECT_PATH}}">
<meta name="robots" content="noindex">
<h1>Redirecting&hellip;</h1>
<a href="{{REDIRECT_PATH}}">Click here if you are not redirected.</a>
</html>

197
compose-db.js
View File

@@ -263,33 +263,69 @@ class DataProcessor {
this.artifacts = {};
}
readExistingData(existingData) {
if (typeof existingData.commits !== "undefined") {
this.commits = existingData.commits;
}
if (typeof existingData.checks !== "undefined") {
this.checks = existingData.checks;
}
if (typeof existingData.runs !== "undefined") {
this.runs = existingData.runs;
}
if (typeof existingData.artifacts !== "undefined") {
this.artifacts = existingData.artifacts;
}
}
processRuns(runsRaw) {
try {
// We will be adding items to the front, so reversing is
// necessary.
runsRaw.reverse();
runsRaw.forEach((item) => {
// Compile basic information about a commit.
let commit = {
"hash": item.oid,
"title": item.messageHeadline,
"committed_date": item.committedDate,
"checks": [],
};
// Check if this commit is already tracked.
let commit = this.commits.find((it) => {
return it.hash === item.oid;
});
if (!commit) {
// Compile basic information about a commit.
commit = {
"hash": item.oid,
"title": item.messageHeadline,
"committed_date": item.committedDate,
"checks": [],
};
this.commits.unshift(commit);
}
const checkSuites = mapNodes(item.checkSuites);
checkSuites.forEach((checkItem) => {
// Compile basic information about a check suite.
let check = {
"check_id": checkItem.databaseId,
"check_url": checkItem.url,
"status": checkItem.status,
"conclusion": checkItem.conclusion,
let check = this.checks[checkItem.databaseId];
"created_at": checkItem.createdAt,
"updated_at": checkItem.updatedAt,
if (typeof check === "undefined") {
// Compile basic information about a check suite.
check = {
"check_id": checkItem.databaseId,
"check_url": checkItem.url,
"status": checkItem.status,
"conclusion": checkItem.conclusion,
"workflow": null,
};
"created_at": checkItem.createdAt,
"updated_at": checkItem.updatedAt,
if (checkItem.workflowRun) {
"workflow": "",
};
this.checks[check.check_id] = check;
} else {
check.status = checkItem.status;
check.conclusion = checkItem.conclusion;
check.updatedAt = checkItem.updatedAt;
}
if (check.workflow === "" && checkItem.workflowRun) {
const runItem = checkItem.workflowRun;
let run = {
"name": runItem.workflow.name,
@@ -303,11 +339,13 @@ class DataProcessor {
check.workflow = run.run_id;
}
this.checks[check.check_id] = check;
commit.checks.push(check.check_id);
});
this.commits.push(commit);
// Existing data may contain this commit, but not all of
// its checks.
if (commit.checks.indexOf(check.check_id) < 0) {
commit.checks.push(check.check_id);
}
});
});
} catch (err) {
console.error(" Error parsing pull request data: " + err);
@@ -315,6 +353,21 @@ class DataProcessor {
}
}
getIncompleteRuns() {
let runs = [];
for (let runId in this.runs) {
const runData = this.runs[runId];
if (runData.artifacts.length > 0) {
continue;
}
runs.push(runId);
}
return runs;
}
processArtifacts(runId, artifactsRaw) {
try {
artifactsRaw.forEach((item) => {
@@ -335,6 +388,37 @@ class DataProcessor {
process.exitCode = ExitCodes.ParseFailure;
}
}
getLatestArtifacts() {
let latest = {};
this.commits.forEach((commit) => {
for (let checkId of commit.checks) {
const check = this.checks[checkId];
if (check.workflow === "") {
continue;
}
const run = this.runs[check.workflow];
run.artifacts.forEach((artifact) => {
if (typeof latest[artifact.name] !== "undefined") {
return; // Continue;
}
latest[artifact.name] = {
"commit_hash": commit.hash,
"check_id": check.check_id,
"workflow_name": run.name,
"artifact_id": artifact.id,
"artifact_name": artifact.name,
"artifact_size": artifact.size,
};
});
}
});
return latest;
}
}
class DataIO {
@@ -369,13 +453,13 @@ class DataIO {
try {
console.log("[*] Loading existing database from a file.");
// const dataPath = `./out/data/${this.data_owner}.${this.data_repo}.${this.data_branch}.json`;
// await fs.access(dataPath, fsConstants.R_OK);
// const existingData = await fs.readFile(dataPath);
const dataPath = `./out/data/${this.data_owner}.${this.data_repo}.${this.data_branch}.json`;
await fs.access(dataPath, fsConstants.R_OK);
const fileRaw = await fs.readFile(dataPath, {encoding: "utf-8"});
return JSON.parse(fileRaw);
} catch (err) {
console.error(" Error loading existing database file: " + err);
process.exitCode = ExitCodes.IOFailure;
return;
return {};
}
}
@@ -392,6 +476,48 @@ class DataIO {
return;
}
}
async createRedirects(artifacts) {
let redirectTemplate = "";
try {
const dataPath = `./build/res/redirect_index.html`;
await fs.access(dataPath, fsConstants.R_OK);
redirectTemplate = await fs.readFile(dataPath, {encoding: "utf-8"});
if (redirectTemplate === "") {
throw new Error("File is missing.");
}
} catch (err) {
console.error(" Error loading a redirect template: " + err);
process.exitCode = ExitCodes.IOFailure;
return;
}
await ensureDir("./out");
await ensureDir("./out/download");
await ensureDir(`./out/download/${this.data_owner}`);
await ensureDir(`./out/download/${this.data_owner}/${this.data_repo}`);
await ensureDir(`./out/download/${this.data_owner}/${this.data_repo}/${this.data_branch}`);
const outputDir = `./out/download/${this.data_owner}/${this.data_repo}/${this.data_branch}`;
for (let artifactName in artifacts) {
await ensureDir(`${outputDir}/${artifactName}`);
try {
const artifact = artifacts[artifactName];
const artifactPath = `https://github.com/godotengine/godot/suites/${artifact.check_id}/artifacts/${artifact.artifact_id}`;
const redirectPage = redirectTemplate.replace(/\{\{REDIRECT_PATH\}\}/g, artifactPath);
await fs.writeFile(`${outputDir}/${artifactName}/index.html`, redirectPage, {encoding: "utf-8"});
console.log(` Created a redirect at ${outputDir}/${artifactName}.`)
} catch (err) {
console.error(` Error saving a redirect page for "${artifactName}": ` + err);
process.exitCode = ExitCodes.IOFailure;
return;
}
}
}
}
function mapNodes(object) {
@@ -455,14 +581,14 @@ async function main() {
dataIO.parseArgs();
checkForExit();
// await dataIO.loadConfig();
// checkForExit();
console.log(`[*] Configured for the "${dataIO.data_owner}/${dataIO.data_repo}" repository; branch ${dataIO.data_branch}.`);
const dataFetcher = new DataFetcher(dataIO.data_owner, dataIO.data_repo);
const dataProcessor = new DataProcessor();
const existingData = await dataIO.loadData();
dataProcessor.readExistingData(existingData);
console.log("[*] Checking the rate limits before.");
await dataFetcher.checkRates();
checkForExit();
@@ -474,7 +600,7 @@ async function main() {
checkForExit();
console.log("[*] Fetching artifact data from GitHub.");
for (let runId in dataProcessor.runs) {
for (let runId of dataProcessor.getIncompleteRuns()) {
const artifactsRaw = await dataFetcher.fetchArtifacts(runId);
checkForExit();
dataProcessor.processArtifacts(runId, artifactsRaw);
@@ -489,6 +615,8 @@ async function main() {
await dataFetcher.checkRates();
checkForExit();
const latestArtifacts = dataProcessor.getLatestArtifacts();
console.log("[*] Finalizing database.")
const output = {
"generated_at": Date.now(),
@@ -496,11 +624,16 @@ async function main() {
"checks": dataProcessor.checks,
"runs": dataProcessor.runs,
"artifacts": dataProcessor.artifacts,
"latest": latestArtifacts,
};
await dataIO.saveData(output, `${dataIO.data_owner}.${dataIO.data_repo}.${dataIO.data_branch}.json`);
checkForExit();
console.log("[*] Creating stable download paths.");
await dataIO.createRedirects(latestArtifacts);
checkForExit();
console.log("[*] Database built.");
}

13
src/paths/index/components/commits/CommitItem.js
View File

@@ -54,9 +54,11 @@ export default class CommitItem extends LitElement {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 8px;
border-bottom: 2px solid var(--g-background-extra-color);
padding: 12px 10px;
}
:host .workflow + .workflow {
border-top: 2px solid var(--g-background-extra-color);
}
:host .workflow-artifacts {
display: flex;
@@ -96,6 +98,13 @@ export default class CommitItem extends LitElement {
@property({ type: String }) repository = '';
render(){
const [...workflows] = this.workflows;
workflows.sort((a,b) => {
if (a.name_sanitized > b.name_sanitized) return 1;
if (a.name_sanitized < b.name_sanitized) return -1;
return 0;
});
return html`
<div class="item-container">
<div class="item-title">
@@ -110,7 +119,7 @@ export default class CommitItem extends LitElement {
</div>
<div class="item-subtitle">${this.title}</div>
<div class="item-workflows">
${this.workflows.map((item) => {
${workflows.map((item) => {
return html`
<div class="workflow">
<div class="workflow-name">${item.name}</div>

89
src/paths/index/components/commits/CommitList.js
View File

@@ -1,6 +1,7 @@
import { LitElement, html, css, customElement, property } from 'lit-element';
import CommitItem from "./CommitItem";
import LatestItem from "./LatestItem";
@customElement('gr-commit-list')
export default class CommitList extends LitElement {
@@ -57,11 +58,60 @@ export default class CommitList extends LitElement {
@property({ type: Object }) checks = {};
@property({ type: Object }) runs = {};
@property({ type: Object }) artifacts = {};
@property({ type: Object }) latest = {};
@property({ type: String }) selectedRepository = "";
@property({ type: String }) selectedBranch = "";
@property({ type: Boolean, reflect: true }) loading = false;
constructor() {
super();
this._workflowsPerCommit = {};
}
_updateWorkflows() {
this._workflowsPerCommit = {};
this.commits.forEach((item) => {
let workflows = [];
for (let checkId in this.checks) {
const check = this.checks[checkId];
if (item.checks.indexOf(check.check_id) < 0) {
continue;
}
if (check.workflow === "" || typeof this.runs[check.workflow] === "undefined") {
continue;
}
const run = this.runs[check.workflow];
if (run.artifacts.length === 0) {
continue;
}
workflows.push({
"name": run.name,
"name_sanitized": run.name.replace(/([^a-zA-Z0-9_\- ]+)/g, "").trim().toLowerCase(),
"check_id": check.check_id,
"artifacts": run.artifacts,
});
}
this._workflowsPerCommit[item.hash] = workflows;
});
}
update(changedProperties) {
// Only recalculate when class properties change; skip for manual updates.
if (changedProperties.size > 0) {
this._updateWorkflows();
}
super.update(changedProperties);
}
render(){
if (this.selectedBranch === "") {
return html``;
@@ -69,42 +119,19 @@ export default class CommitList extends LitElement {
if (this.loading) {
return html`
<span class="branch-commits-empty">Loading artifacts...</span>
`
`;
}
return html`
<div class="branch-commits">
<gr-latest-item
.artifacts="${this.latest}"
.repository="${this.selectedRepository}"
.branch="${this.selectedBranch}"
></gr-latest-item>
${this.commits.map((item) => {
let workflows = [];
for (let checkId in this.checks) {
const check = this.checks[checkId];
if (item.checks.indexOf(check.check_id) < 0) {
continue;
}
if (check.workflow == null || typeof this.runs[check.workflow] === "undefined") {
continue;
}
const run = this.runs[check.workflow];
if (run.artifacts.length === 0) {
continue;
}
workflows.push({
"name": run.name,
"name_sanitized": run.name.replace(/([^a-zA-Z0-9_\- ]+)/g, "").trim().toLowerCase(),
"check_id": check.check_id,
"artifacts": run.artifacts,
});
}
workflows.sort((a,b) => {
if (a.name_sanitized > b.name_sanitized) return 1;
if (a.name_sanitized < b.name_sanitized) return -1;
return 0;
});
const workflows = this._workflowsPerCommit[item.hash];
return html`
<gr-commit-item

174
src/paths/index/components/commits/LatestItem.js Normal file
View File

@@ -0,0 +1,174 @@
import { LitElement, html, css, customElement, property } from 'lit-element';
@customElement('gr-latest-item')
export default class LatestItem extends LitElement {
static get styles() {
return css`
/** Colors and variables **/
:host {
--item-border-color: #fcfcfa;
}
@media (prefers-color-scheme: dark) {
:host {
--item-border-color: #0d1117;
}
}
/** Component styling **/
:host {
border-bottom: 3px solid var(--item-border-color);
display: block;
padding: 14px 12px 20px 12px;
}
:host a {
color: var(--link-font-color);
text-decoration: none;
}
:host a:hover {
color: var(--link-font-color-hover);
}
:host .item-title {
display: inline-flex;
justify-content: space-between;
font-size: 20px;
margin-top: 6px;
margin-bottom: 12px;
width: 100%;
}
:host .item-subtitle {
color: var(--dimmed-font-color);
font-size: 16px;
line-height: 20px;
word-break: break-word;
}
:host .item-workflows {
margin-top: 12px;
}
:host .workflow {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 8px;
padding: 12px 10px;
}
:host .workflow + .workflow {
border-top: 2px solid var(--g-background-extra-color);
}
:host .workflow-artifacts {
display: flex;
flex-direction: column;
gap: 6px;
color: var(--dimmed-font-color);
font-size: 14px;
}
:host .workflow-artifacts a {
font-size: 15px;
font-weight: 600;
}
@media only screen and (max-width: 900px) {
:host {
padding: 14px 0 20px 0;
}
:host .workflow {
grid-template-columns: 1fr;
}
}
@media only screen and (max-width: 640px) {
:host .item-container {
padding: 0 10px;
}
}
`;
}
@property({ type: Object }) artifacts = {};
@property({ type: String }) repository = '';
@property({ type: String }) branch = '';
constructor() {
super();
this._latestByWorkflow = [];
}
_updateWorkflows() {
this._latestByWorkflow = [];
const existingWorkflow = {};
for (let artifactName in this.artifacts) {
const artifact = this.artifacts[artifactName];
if (typeof existingWorkflow[artifact.workflow_name] === "undefined") {
existingWorkflow[artifact.workflow_name] = {
"name": artifact.workflow_name,
"name_sanitized": artifact.workflow_name.replace(/([^a-zA-Z0-9_\- ]+)/g, "").trim().toLowerCase(),
"artifacts": [],
};
this._latestByWorkflow.push(existingWorkflow[artifact.workflow_name]);
}
existingWorkflow[artifact.workflow_name].artifacts.push(artifact);
}
this._latestByWorkflow.sort((a,b) => {
if (a.name_sanitized > b.name_sanitized) return 1;
if (a.name_sanitized < b.name_sanitized) return -1;
return 0;
});
}
update(changedProperties) {
// Only recalculate when class properties change; skip for manual updates.
if (changedProperties.size > 0) {
this._updateWorkflows();
}
super.update(changedProperties);
}
render(){
return html`
<div class="item-container">
<div class="item-title">
<span>Latest</span>
</div>
<div class="item-subtitle">Builds may be from different runs, depending on their availability.</div>
<div class="item-workflows">
${this._latestByWorkflow.map((item) => {
return html`
<div class="workflow">
<div class="workflow-name">${item.name}</div>
<div class="workflow-artifacts">
${item.artifacts.map((artifact) => {
return html`
<span>
<a
href="/download/${this.repository}/${this.branch}/${artifact.artifact_name}"
target="_blank"
>
${artifact.artifact_name}
</a>
<span>(${greports.format.humanizeBytes(artifact.artifact_size)})</span>
</span>
`;
})}
</div>
</div>
`;
})}
</div>
</div>
`;
}
}

3
src/paths/index/entry.js
View File

@@ -120,6 +120,7 @@ export default class EntryComponent extends LitElement {
let checks = {};
let runs = {};
let artifacts = {};
let latest = {};
if (this._selectedBranch !== "" && typeof this._branchData[this._selectedBranch] !== "undefined") {
const branchData = this._branchData[this._selectedBranch];
@@ -128,6 +129,7 @@ export default class EntryComponent extends LitElement {
checks = branchData.checks;
runs = branchData.runs;
artifacts = branchData.artifacts;
latest = branchData.latest;
}
return html`
@@ -153,6 +155,7 @@ export default class EntryComponent extends LitElement {
.checks="${checks}"
.runs="${runs}"
.artifacts="${artifacts}"
.latest="${latest}"
.selectedRepository="${this._selectedRepository}"
.selectedBranch="${this._selectedBranch}"