wehub-resource-sync
277 lines
7.9 KiB
Django/Jinja
277 lines
7.9 KiB
Django/Jinja
def per_exec_ws(folder) {
|
|
return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
|
|
}
|
|
|
|
// initialize source codes
|
|
def init_git() {
|
|
retry(5) {
|
|
checkout scm
|
|
}
|
|
|
|
// Add more info about job node
|
|
sh (
|
|
script: './tests/scripts/task_show_node_info.sh',
|
|
label: 'Show executor node info',
|
|
)
|
|
|
|
// Determine merge commit to use for all stages
|
|
if (env.CHANGE_TARGET) {
|
|
// This is a PR build, so merge with the latest of the PR's target branch
|
|
// (e.g. main or a release branch like v0.25.0).
|
|
merge_with_target()
|
|
} else {
|
|
// This is a branch build (main or a release branch). Only set
|
|
// upstream_revision to HEAD and skip merging to avoid a race with another
|
|
// commit merged to the branch.
|
|
update_upstream_revision("HEAD")
|
|
}
|
|
|
|
sh(
|
|
script: """
|
|
set -eux
|
|
. ${jenkins_scripts_root}/retry.sh
|
|
retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
|
|
""",
|
|
label: 'Update git submodules',
|
|
)
|
|
checkout_trusted_files()
|
|
}
|
|
|
|
def update_upstream_revision(git_ref) {
|
|
if (upstream_revision == null) {
|
|
upstream_revision = sh(
|
|
script: "git log -1 ${git_ref} --format=\'%H\'",
|
|
label: 'Determine upstream revision',
|
|
returnStdout: true,
|
|
).trim()
|
|
}
|
|
}
|
|
|
|
def merge_with_target() {
|
|
def target = env.CHANGE_TARGET
|
|
sh (
|
|
script: "git fetch origin ${target}",
|
|
label: "Fetch target branch ${target}",
|
|
)
|
|
update_upstream_revision("FETCH_HEAD")
|
|
sh (
|
|
script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
|
|
label: "Merge to origin/${target}"
|
|
)
|
|
}
|
|
|
|
def docker_init(image) {
|
|
// Clear out all Docker images that aren't going to be used
|
|
sh(
|
|
script: """
|
|
set -eux
|
|
docker image ls --all
|
|
IMAGES=\$(docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %})
|
|
|
|
echo -e "Found images:\\n\$IMAGES"
|
|
echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
|
|
|
|
docker image ls --all
|
|
""",
|
|
label: 'Clean old Docker images',
|
|
)
|
|
|
|
if (image.contains("amazonaws.com")) {
|
|
// If this string is in the image name it's from ECR and needs to be pulled
|
|
// with the right credentials
|
|
ecr_pull(image)
|
|
} else {
|
|
sh(
|
|
script: """
|
|
set -eux
|
|
. ${jenkins_scripts_root}/retry.sh
|
|
retry 5 docker pull ${image}
|
|
""",
|
|
label: 'Pull docker image',
|
|
)
|
|
}
|
|
}
|
|
|
|
def ecr_pull(full_name) {
|
|
aws_account_id = sh(
|
|
returnStdout: true,
|
|
script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
|
|
label: 'Get AWS ID'
|
|
).trim()
|
|
|
|
try {
|
|
withEnv([
|
|
"AWS_ACCOUNT_ID=${aws_account_id}",
|
|
'AWS_DEFAULT_REGION={{ aws_default_region }}',
|
|
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
|
|
sh(
|
|
script: '''
|
|
set -eux
|
|
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
|
|
''',
|
|
label: 'Log in to ECR'
|
|
)
|
|
sh(
|
|
script: """
|
|
set -eux
|
|
. ${jenkins_scripts_root}/retry.sh
|
|
retry 5 docker pull ${full_name}
|
|
""",
|
|
label: 'Pull image from ECR'
|
|
)
|
|
}
|
|
} finally {
|
|
withEnv([
|
|
"AWS_ACCOUNT_ID=${aws_account_id}",
|
|
'AWS_DEFAULT_REGION={{ aws_default_region }}',
|
|
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
|
|
sh(
|
|
script: 'docker logout $AWS_ECR_REPO',
|
|
label: 'Clean up login credentials'
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
def should_skip_slow_tests(pr_number) {
|
|
withCredentials([string(
|
|
credentialsId: 'tvm-bot-jenkins-reader',
|
|
variable: 'GITHUB_TOKEN',
|
|
)]) {
|
|
// Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
|
|
result = sh (
|
|
returnStatus: true,
|
|
script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
|
|
label: 'Check if CI should run slow tests',
|
|
)
|
|
}
|
|
return result == 0
|
|
}
|
|
|
|
def cancel_previous_build() {
|
|
// cancel previous build if it is not on main.
|
|
if (env.BRANCH_NAME != 'main') {
|
|
def buildNumber = env.BUILD_NUMBER as int
|
|
// Milestone API allows us to cancel previous build
|
|
// with the same milestone number
|
|
if (buildNumber > 1) milestone(buildNumber - 1)
|
|
milestone(buildNumber)
|
|
}
|
|
}
|
|
|
|
def is_last_build() {
|
|
// check whether it is last build
|
|
try {
|
|
return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number
|
|
} catch (Throwable ex) {
|
|
echo 'Error during check is_last_build ' + ex.toString()
|
|
return false
|
|
}
|
|
}
|
|
|
|
def checkout_trusted_files() {
|
|
// trust everything from branch builds
|
|
if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
|
|
return;
|
|
}
|
|
|
|
// trust peoople listed in CONTRIBUTING.md
|
|
grep_code = sh(
|
|
returnStatus: true,
|
|
script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
|
|
label: 'Check if change is from a contributor',
|
|
)
|
|
|
|
if (grep_code == 1) {
|
|
// Any scripts that run on the bare host and not inside a Docker container
|
|
// (especially those that access secrets) should be checked out here so
|
|
// only trusted versions are used in CI
|
|
sh(
|
|
script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
|
|
label: 'Check out trusted files',
|
|
)
|
|
}
|
|
}
|
|
|
|
def should_skip_ci(pr_number) {
|
|
if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
|
|
// never skip CI on build sourced from a branch
|
|
return false
|
|
}
|
|
glob_skip_ci_code = sh (
|
|
returnStatus: true,
|
|
script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
|
|
label: 'Check if CI should be skipped due to changed files',
|
|
)
|
|
if (glob_skip_ci_code == 0) {
|
|
return true
|
|
}
|
|
withCredentials([string(
|
|
credentialsId: 'tvm-bot-jenkins-reader',
|
|
variable: 'GITHUB_TOKEN',
|
|
)]) {
|
|
// Exit code of 1 means run full CI (or the script had an error, so run
|
|
// full CI just in case). Exit code of 0 means skip CI.
|
|
git_skip_ci_code = sh (
|
|
returnStatus: true,
|
|
script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
|
|
label: 'Check if CI should be skipped',
|
|
)
|
|
}
|
|
return git_skip_ci_code == 0
|
|
}
|
|
|
|
def prepare(node_type) {
|
|
stage('Prepare') {
|
|
node(node_type) {
|
|
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
|
|
init_git()
|
|
|
|
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
|
|
sh(
|
|
script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }} {% endfor %}",
|
|
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
|
|
)
|
|
// Pull image names from the results of should_rebuild_docker.py
|
|
{% for image in images %}
|
|
{{ image.name }} = sh(
|
|
script: "cat .docker-image-names/{{ image.name }}",
|
|
label: "Find docker image name for {{ image.name }}",
|
|
returnStdout: true,
|
|
).trim()
|
|
{% endfor %}
|
|
}
|
|
|
|
{% for image in images %}
|
|
{{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }}
|
|
{% endfor %}
|
|
|
|
sh (script: """
|
|
echo "Docker images being used in this build:"
|
|
{% for image in images %}
|
|
echo " {{ image.name }} = ${ {{- image.name -}} }"
|
|
{% endfor %}
|
|
""", label: 'Docker image names')
|
|
|
|
is_docs_only_build = sh (
|
|
returnStatus: true,
|
|
script: "./${jenkins_scripts_root}/git_change_docs.sh",
|
|
label: 'Check for docs only changes',
|
|
)
|
|
skip_ci = should_skip_ci(env.CHANGE_ID)
|
|
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
|
|
rebuild_docker_images = sh (
|
|
returnStatus: true,
|
|
script: "./${jenkins_scripts_root}/git_change_docker.sh",
|
|
label: 'Check for any docker changes',
|
|
)
|
|
|
|
if (skip_ci) {
|
|
// Don't rebuild when skipping CI
|
|
rebuild_docker_images = false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|