chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
const { readFileSync, existsSync, readdirSync, statSync } = require("fs");
|
||||
const { join, basename } = require("path");
|
||||
|
||||
// Constants
|
||||
const RELEASE_TAG = "nightly";
|
||||
const DAYS_TO_KEEP = 3;
|
||||
|
||||
/**
|
||||
* Check if artifact file type is supported
|
||||
*/
|
||||
function isSupportedArtifact(filename) {
|
||||
return /\.(whl|jar|tar\.gz)$/.test(filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get content type based on file extension
|
||||
*/
|
||||
function getContentType(filename) {
|
||||
if (filename.match(/\.whl$/)) {
|
||||
return "application/zip";
|
||||
} else if (filename.match(/\.tar\.gz$/)) {
|
||||
return "application/gzip";
|
||||
} else if (filename.match(/\.jar$/)) {
|
||||
return "application/java-archive";
|
||||
}
|
||||
throw new Error(
|
||||
`Unsupported file type for content type: ${filename}. Only .whl, .jar, and .tar.gz are supported.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if asset should be deleted based on age
|
||||
*/
|
||||
function shouldDeleteAsset(asset, daysToKeep) {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - daysToKeep);
|
||||
const assetDate = new Date(asset.created_at);
|
||||
return assetDate < cutoffDate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add commit SHA to filename based on file type
|
||||
*/
|
||||
function addShaToFilename(filename, sha) {
|
||||
const shortSha = sha.substring(0, 7);
|
||||
|
||||
// Match .whl files
|
||||
if (filename.match(/\.whl$/)) {
|
||||
// For wheel files, insert SHA as build tag before .whl extension
|
||||
// Build tags must start with a digit, so prefix with "0"
|
||||
const wheelParts = filename.match(/^(.+?)(-py[^-]+(?:-[^-]+)*\.whl)$/);
|
||||
if (wheelParts) {
|
||||
return `${wheelParts[1]}-0${shortSha}${wheelParts[2]}`;
|
||||
}
|
||||
// Fallback for non-standard wheel names
|
||||
return filename.replace(/\.whl$/, `-0${shortSha}.whl`);
|
||||
}
|
||||
|
||||
// Match .jar files
|
||||
if (filename.match(/\.jar$/)) {
|
||||
return filename.replace(/\.jar$/, `-${shortSha}.jar`);
|
||||
}
|
||||
|
||||
// Match .tar.gz files
|
||||
if (filename.match(/\.tar\.gz$/)) {
|
||||
return filename.replace(/\.tar\.gz$/, `-${shortSha}.tar.gz`);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unexpected file extension for: ${filename}. Only .whl, .jar, and .tar.gz are supported.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload artifacts to a GitHub release
|
||||
*/
|
||||
async function uploadSnapshots({ github, context, artifactDir }) {
|
||||
if (!existsSync(artifactDir)) {
|
||||
throw new Error(`Artifacts directory not found: ${artifactDir}`);
|
||||
}
|
||||
|
||||
const artifactFiles = readdirSync(artifactDir)
|
||||
.map((f) => join(artifactDir, f))
|
||||
.filter((f) => statSync(f).isFile());
|
||||
|
||||
if (artifactFiles.length === 0) {
|
||||
throw new Error(`No artifacts found in ${artifactDir}`);
|
||||
}
|
||||
|
||||
// Check for unsupported file types
|
||||
const unsupportedFiles = artifactFiles.filter((f) => !isSupportedArtifact(f));
|
||||
if (unsupportedFiles.length > 0) {
|
||||
const names = unsupportedFiles.map((f) => ` - ${basename(f)}`).join("\n");
|
||||
throw new Error(
|
||||
`Found unsupported file types:\n${names}\nOnly .whl, .jar, and .tar.gz files are supported.`
|
||||
);
|
||||
}
|
||||
|
||||
// Check if the release already exists
|
||||
const { owner, repo } = context.repo;
|
||||
let release;
|
||||
let releaseExists = false;
|
||||
try {
|
||||
const { data } = await github.rest.repos.getReleaseByTag({
|
||||
owner,
|
||||
repo,
|
||||
tag: RELEASE_TAG,
|
||||
});
|
||||
release = data;
|
||||
releaseExists = true;
|
||||
console.log(`Found existing release: ${release.id}`);
|
||||
} catch (error) {
|
||||
if (error.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const releaseParams = {
|
||||
owner,
|
||||
repo,
|
||||
tag_name: RELEASE_TAG,
|
||||
target_commitish: context.sha,
|
||||
name: `Nightly Build ${new Date().toISOString().split("T")[0]}`,
|
||||
body: `This is an automated nightly build of MLflow.
|
||||
|
||||
**Last updated:** ${new Date().toUTCString()}
|
||||
**Commit:** ${context.sha}
|
||||
|
||||
**Note:** This release is automatically updated daily with the latest changes from the master branch.`,
|
||||
prerelease: true,
|
||||
make_latest: "false",
|
||||
};
|
||||
|
||||
if (releaseExists) {
|
||||
console.log("Updating existing nightly release...");
|
||||
const { data: updatedRelease } = await github.rest.repos.updateRelease({
|
||||
...releaseParams,
|
||||
release_id: release.id,
|
||||
});
|
||||
release = updatedRelease;
|
||||
console.log(`Updated existing release: ${release.id}`);
|
||||
} else {
|
||||
console.log("Creating new nightly release...");
|
||||
const { data: newRelease } = await github.rest.repos.createRelease(releaseParams);
|
||||
release = newRelease;
|
||||
console.log(`Created new release: ${release.id}`);
|
||||
}
|
||||
|
||||
console.log("Fetching all existing assets...");
|
||||
const allAssets = await github.paginate(github.rest.repos.listReleaseAssets, {
|
||||
owner,
|
||||
repo,
|
||||
release_id: release.id,
|
||||
});
|
||||
console.log(`Found ${allAssets.length} existing assets`);
|
||||
|
||||
// Delete old assets.
|
||||
for (const asset of allAssets) {
|
||||
if (shouldDeleteAsset(asset, DAYS_TO_KEEP)) {
|
||||
const assetDate = new Date(asset.created_at).toISOString().split("T")[0];
|
||||
console.log(`Deleting old asset (created ${assetDate}): ${asset.name}`);
|
||||
await github.rest.repos.deleteReleaseAsset({
|
||||
owner,
|
||||
repo,
|
||||
asset_id: asset.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Filter to get remaining assets after deletion
|
||||
const remainingAssets = allAssets.filter((asset) => !shouldDeleteAsset(asset, DAYS_TO_KEEP));
|
||||
|
||||
// Upload all artifacts
|
||||
for (const artifactPath of artifactFiles) {
|
||||
const artifactName = basename(artifactPath);
|
||||
const contentType = getContentType(artifactName);
|
||||
const nameWithSha = addShaToFilename(artifactName, context.sha);
|
||||
|
||||
// Check if artifact with SHA already exists in remaining assets
|
||||
if (remainingAssets.some((asset) => asset.name === nameWithSha)) {
|
||||
console.log(`Artifact already exists: ${nameWithSha} (skipping upload)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`Uploading ${artifactName} as ${nameWithSha}...`);
|
||||
const artifactData = readFileSync(artifactPath);
|
||||
await github.rest.repos.uploadReleaseAsset({
|
||||
owner,
|
||||
repo,
|
||||
release_id: release.id,
|
||||
name: nameWithSha,
|
||||
data: artifactData,
|
||||
headers: {
|
||||
"content-type": contentType,
|
||||
"content-length": artifactData.length,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Successfully uploaded ${artifactName} as ${nameWithSha}`);
|
||||
}
|
||||
|
||||
console.log("All artifacts uploaded successfully");
|
||||
}
|
||||
|
||||
module.exports = { uploadSnapshots };
|
||||
Reference in New Issue
Block a user