chore: import upstream snapshot with attribution

ci/scripts/github/__init__.py

@@ -0,0 +1,18 @@
+# isort: skip_file
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Package to enable testing of GitHub scripts."""

ci/scripts/github/github_tvmbot.py

@@ -0,0 +1,799 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E402, E501, F841, RUF012
+
+import argparse
+import json
+import logging
+import os
+import re
+import sys
+import traceback
+import warnings
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
+
+from cmd_utils import init_log
+from git_utils import GitHubRepo, git, parse_remote, post
+
+Review = dict[str, Any]
+CIJob = dict[str, Any]
+Comment = dict[str, Any]
+CommentChecker = Callable[[Comment], bool]
+
+EXPECTED_JOBS = ["tvm-ci/pr-head"]
+TVM_BOT_JENKINS_TOKEN = os.environ["TVM_BOT_JENKINS_TOKEN"]
+GH_ACTIONS_TOKEN = os.environ["GH_ACTIONS_TOKEN"]
+JENKINS_URL = "https://ci.tlcpack.ai/"
+THANKS_MESSAGE = r"(\s*)Thanks for contributing to TVM!   Please refer to guideline https://tvm.apache.org/docs/contribute/ for useful information and tips. After the pull request is submitted, please request code reviews from \[Reviewers\]\(https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers\) by  them in the pull request thread.(\s*)"
+
+
+def to_json_str(obj: Any) -> str:
+    return json.dumps(obj, indent=2)
+
+
+COLLABORATORS_QUERY = """
+query ($owner: String!, $name: String!, $user: String!) {
+  repository(owner: $owner, name: $name) {
+    collaborators(query: $user, first: 100) {
+      nodes {
+        login
+      }
+    }
+  }
+}
+"""
+
+MENTIONABLE_QUERY = """
+query ($owner: String!, $name: String!, $user: String!) {
+  repository(owner: $owner, name: $name) {
+    mentionableUsers(query: $user, first: 100) {
+      nodes {
+        login
+      }
+    }
+  }
+}
+"""
+
+
+PR_QUERY = """
+    query ($owner: String!, $name: String!, $number: Int!) {
+      repository(owner: $owner, name: $name) {
+        pullRequest(number: $number) {
+          title
+          body
+          state
+          author {
+            login
+          }
+          comments(last: 100) {
+            pageInfo {
+              hasPreviousPage
+            }
+            nodes {
+              authorAssociation
+              author {
+                login
+              }
+              id
+              updatedAt
+              body
+            }
+          }
+          authorCommits:commits(last:100) {
+            nodes {
+              commit {
+                authors(first:100) {
+                  nodes {
+                    name
+                    email
+                  }
+                }
+              }
+            }
+          }
+          commits(last: 1) {
+            nodes {
+              commit {
+                oid
+                statusCheckRollup {
+                  contexts(first: 100) {
+                    pageInfo {
+                      hasNextPage
+                    }
+                    nodes {
+                      ... on CheckRun {
+                        name
+                        databaseId
+                        checkSuite {
+                          workflowRun {
+                            databaseId
+                            workflow {
+                              name
+                            }
+                          }
+                        }
+                        status
+                        conclusion
+                        url
+                      }
+                      ... on StatusContext {
+                        state
+                        context
+                        targetUrl
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+          reviewDecision
+          reviews(last: 100) {
+            pageInfo {
+              hasPreviousPage
+            }
+            nodes {
+              body
+              updatedAt
+              url
+              id
+              authorCanPushToRepository
+              commit {
+                oid
+              }
+              author {
+                login
+              }
+              state
+            }
+          }
+        }
+      }
+    }
+    """
+
+
+def walk(obj, visitor, parent_key=None):
+    """
+    Recursively call 'visitor' on all the children of a dictionary
+    """
+    visitor(obj, parent_key)
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            walk(v, visitor, parent_key=k)
+    elif isinstance(obj, list):
+        for v in obj:
+            walk(v, visitor)
+
+
+class PR:
+    def __init__(
+        self,
+        number: int,
+        owner: str,
+        repo: str,
+        dry_run: bool = False,
+        raw_data: dict[str, Any] | None = None,
+    ):
+        self.owner = owner
+        self.number = number
+        self.repo_name = repo
+        self.dry_run = dry_run
+        self.has_error = False
+
+        if dry_run and raw_data:
+            # In test mode there is no need to fetch anything
+            self.raw = raw_data
+            self.github = None
+        else:
+            self.github = GitHubRepo(user=owner, repo=repo, token=os.environ["GITHUB_TOKEN"])
+            if os.getenv("DEBUG", "0") == "1":
+                # For local runs fill in the requested data but cache it for
+                # later use
+                cached_path = Path("pr.json")
+                if not cached_path.exists():
+                    self.raw = self.fetch_data()
+                    with open(cached_path, "w") as f:
+                        json.dump(self.raw, f, indent=2)
+                else:
+                    with open(cached_path) as f:
+                        self.raw = json.load(f)
+            else:
+                # Usual path, fetch the PR's data based on the number from
+                # GitHub
+                self.raw = self.fetch_data()
+
+        def checker(obj, parent_key):
+            """
+            Verify that any paged results don't have extra data (if so the bot
+            may still work since most relevant comments will be more recent)
+            """
+            if parent_key == "pageInfo":
+                if obj.get("hasPreviousPage", False):
+                    warnings.warn(f"Found {obj} with a previous page, bot may be missing data")
+                if obj.get("hasNextPage", False):
+                    warnings.warn(f"Found {obj} with a next page, bot may be missing data")
+
+        walk(self.raw, checker)
+
+        logging.info(f"Verified data, running with PR {to_json_str(self.raw)}")
+
+    def __repr__(self):
+        return json.dumps(self.raw, indent=2)
+
+    def react(self, comment: dict[str, Any], content: str):
+        """
+        React with a thumbs up to a comment
+        """
+        url = f"issues/comments/{comment['id']}/reactions"
+        data = {"content": content}
+        if self.dry_run:
+            logging.info(f"Dry run, would have +1'ed to {url} with {data}")
+        else:
+            self.github.post(url, data=data)
+
+    def head_commit(self):
+        return self.raw["commits"]["nodes"][0]["commit"]
+
+    def co_authors(self) -> list[str]:
+        authors = []
+        for commit in self.raw["authorCommits"]["nodes"]:
+            # Co-authors always come after the main author according to the
+            # GitHub docs, so ignore the first item
+            for author in commit["commit"]["authors"]["nodes"][1:]:
+                name = author["name"]
+                email = author["email"]
+                authors.append(f"{name} <{email}>")
+
+        return list(set(authors))
+
+    def head_oid(self):
+        return self.head_commit()["oid"]
+
+    def ci_jobs(self) -> list[CIJob]:
+        """
+        Get a list of all CI jobs (GitHub Actions and other) in a unified format
+        """
+        jobs = []
+        for item in self.head_commit()["statusCheckRollup"]["contexts"]["nodes"]:
+            if "checkSuite" in item:
+                # GitHub Actions job, parse separately
+                status = item["conclusion"]
+                if status is None:
+                    # If the 'conclusion' isn't filled out the job hasn't
+                    # finished yet
+                    status = "PENDING"
+                workflow_name = item["checkSuite"]["workflowRun"]["workflow"]["name"]
+                if workflow_name != "CI":
+                    # Ignore all jobs that aren't in the main.yml workflow (these are mostly
+                    # automation jobs that run on PRs for tagging / reviews)
+                    continue
+                check_name = item["name"]
+                jobs.append(
+                    {
+                        "name": f"{workflow_name} / {check_name}",
+                        "url": item["url"],
+                        "status": status.upper(),
+                    }
+                )
+            else:
+                # GitHub Status (e.g. from Jenkins)
+                jobs.append(
+                    {
+                        "name": item["context"],
+                        "url": item["targetUrl"],
+                        "status": item["state"].upper(),
+                    }
+                )
+
+        logging.info(f"Found CI jobs for {self.head_commit()['oid']} {to_json_str(jobs)}")
+        return jobs
+
+    def reviews(self) -> list[Review]:
+        return self.raw["reviews"]["nodes"]
+
+    def head_commit_reviews(self) -> list[Review]:
+        """
+        Find reviews associated with the head commit
+        """
+        commits_to_review_status: dict[str, list[Review]] = {}
+
+        for review in self.reviews():
+            if not review["authorCanPushToRepository"]:
+                # ignore reviews from non-committers
+                continue
+
+            oid = review["commit"]["oid"]
+            if oid in commits_to_review_status:
+                commits_to_review_status[oid].append(review)
+            else:
+                commits_to_review_status[oid] = [review]
+
+        # Only use the data for the head commit of the PR
+        head_reviews = commits_to_review_status.get(self.head_oid(), [])
+        return head_reviews
+
+    def fetch_data(self):
+        """
+        Fetch the data for this PR from GitHub
+        """
+        return self.github.graphql(
+            query=PR_QUERY,
+            variables={
+                "owner": self.owner,
+                "name": self.repo_name,
+                "number": self.number,
+            },
+        )["data"]["repository"]["pullRequest"]
+
+    def search_collaborator(self, user: str) -> list[dict[str, Any]]:
+        """
+        Query GitHub for collaborators matching 'user'
+        """
+        return self.search_users(user, COLLABORATORS_QUERY)["collaborators"]["nodes"]
+
+    def search_users(self, user: str, query: str) -> list[dict[str, Any]]:
+        return self.github.graphql(
+            query=query,
+            variables={
+                "owner": self.owner,
+                "name": self.repo_name,
+                "user": user,
+            },
+        )["data"]["repository"]
+
+    def search_mentionable_users(self, user: str) -> list[dict[str, Any]]:
+        return self.search_users(user, MENTIONABLE_QUERY)["mentionableUsers"]["nodes"]
+
+    def comment(self, text: str) -> None:
+        """
+        Leave the comment 'text' on this PR
+        """
+        logging.info(f"Commenting:\n{text}")
+        # TODO: Update latest comment in-place if there has been no activity
+        data = {"body": text}
+        url = f"issues/{self.number}/comments"
+        if self.dry_run:
+            logging.info(
+                f"Dry run, would have commented on url={url} commenting with data={to_json_str(data)}"
+            )
+            return
+
+        self.github.post(url, data=data)
+
+    def state(self) -> str:
+        """
+        PR state (OPEN, CLOSED, MERGED, etc)
+        """
+        return self.raw["state"]
+
+    def processed_body(self) -> str:
+        body = self.raw["body"].strip().replace("\r", "")
+        # Remove any @-mentions of people
+        body = re.sub(r"(\s)@", r"\g<1>", body)
+
+        # Remove the auto-inserted text since it's not useful to have in the commit log
+        body = re.sub(THANKS_MESSAGE, "\n\n", body)
+        return body.strip()
+
+    def body_with_co_authors(self) -> str:
+        """
+        Add 'Co-authored-by' strings to the PR body based on the prior commits
+        in the PR
+        """
+        body = self.processed_body()
+        author_lines = self.co_authors()
+        logging.info(f"Found co-authors: author_lines={author_lines}")
+        full_author_lines = [f"Co-authored-by: {author_line}" for author_line in author_lines]
+
+        authors_to_add = []
+        for author_line in author_lines:
+            if author_line not in body:
+                authors_to_add.append(f"Co-authored-by: {author_line}")
+
+        if len(authors_to_add) > 0:
+            # If the line isn't already in the PR body (it could have been
+            # added manually), put it in
+            full_author_text = "\n".join(authors_to_add)
+            body = f"{body}\n\n{full_author_text}"
+
+        return body
+
+    def merge(self) -> None:
+        """
+        Request a merge of this PR via the GitHub API
+        """
+        url = f"pulls/{self.number}/merge"
+
+        title = self.raw["title"] + f" (#{self.number})"
+        body = self.body_with_co_authors()
+        logging.info(f"Full commit:\n{title}\n\n{body}")
+
+        data = {
+            "commit_title": title,
+            "commit_message": body,
+            # The SHA is necessary in case there was an update right when this
+            # script ran, GitHub will sort out who won
+            "sha": self.head_oid(),
+            "merge_method": "squash",
+        }
+        if self.dry_run:
+            logging.info(f"Dry run, would have merged with url={url} and data={to_json_str(data)}")
+            return
+
+        r = self.github.put(url, data=data)
+        logging.info(f"GitHub merge response: {r}")
+        return r
+
+    def author(self) -> str:
+        return self.raw["author"]["login"]
+
+    def find_failed_ci_jobs(self) -> list[CIJob]:
+        # NEUTRAL is GitHub Action's way of saying cancelled
+        return [
+            job
+            for job in self.ci_jobs()
+            if job["status"] not in {"SUCCESS", "SUCCESSFUL", "SKIPPED"}
+        ]
+
+    def find_missing_expected_jobs(self) -> list[str]:
+        # Map of job name: has seen in completed jobs
+        seen_expected_jobs = {name: False for name in EXPECTED_JOBS}
+        logging.info(f"Expected to see jobs: {seen_expected_jobs}")
+
+        missing_expected_jobs = []
+        for job in self.ci_jobs():
+            seen_expected_jobs[job["name"]] = True
+
+        for name, seen in seen_expected_jobs.items():
+            if not seen:
+                missing_expected_jobs.append(name)
+
+        return missing_expected_jobs
+
+    def trigger_gha_ci(self, sha: str) -> None:
+        logging.info("POST-ing a workflow_dispatch event to main.yml")
+        actions_github = GitHubRepo(
+            user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN
+        )
+        r = actions_github.post(
+            url="actions/workflows/main.yml/dispatches",
+            data={
+                "ref": "main",
+            },
+        )
+        logging.info(f"Successful workflow_dispatch: {r}")
+
+    def merge_if_passed_checks(self) -> dict[str, Any] | None:
+        failed_ci_jobs = self.find_failed_ci_jobs()
+        all_ci_passed = len(failed_ci_jobs) == 0
+        has_one_approval = False
+
+        if not all_ci_passed:
+            failed_jobs_msg = "\n".join(
+                [f" * [{job['name']} (`{job['status']}`)]({job['url']})" for job in failed_ci_jobs]
+            )
+            self.comment(
+                f"Cannot merge, these CI jobs are not successful on {self.head_oid()}:\n{failed_jobs_msg}"
+            )
+            return None
+
+        missing_expected_jobs = self.find_missing_expected_jobs()
+
+        if len(missing_expected_jobs) > 0:
+            missing_jobs_msg = "\n".join([f" * `{name}`" for name in missing_expected_jobs])
+            self.comment(f"Cannot merge, missing expected jobs:\n{missing_jobs_msg}")
+            return None
+
+        head_commit_reviews = self.head_commit_reviews()
+        for review in head_commit_reviews:
+            if review["state"] == "CHANGES_REQUESTED":
+                self.comment(
+                    f"Cannot merge, found [this review]({review['url']}) on {self.head_oid()} with changes requested"
+                )
+                return None
+
+            if review["state"] == "APPROVED":
+                has_one_approval = True
+                logging.info(f"Found approving review: {to_json_str(review)}")
+
+        if has_one_approval and all_ci_passed:
+            return self.merge()
+        elif not has_one_approval:
+            self.comment(
+                f"Cannot merge, did not find any approving reviews from users with write access on {self.head_oid()}"
+            )
+            return None
+        elif not all_ci_passed:
+            self.comment(f"Cannot merge, CI did not pass on on {self.head_oid()}")
+            return None
+
+    def rerun_jenkins_ci(self) -> None:
+        job_names = [
+            "tvm-arm",
+            "tvm-cpu",
+            "tvm-docker",
+            "tvm-gpu",
+            "tvm-wasm",
+        ]
+        for name in job_names:
+            url = JENKINS_URL + f"job/{name}/job/PR-{self.number}/buildWithParameters"
+            logging.info(f"Rerunning ci with URL={url}")
+            if self.dry_run:
+                logging.info("Dry run, not sending POST")
+            else:
+                post(url, auth=("tvm-bot", TVM_BOT_JENKINS_TOKEN))
+
+    def rerun_github_actions(self) -> None:
+        workflow_ids = []
+        for item in self.head_commit()["statusCheckRollup"]["contexts"]["nodes"]:
+            if "checkSuite" in item and item["conclusion"] == "FAILURE":
+                workflow_id = item["checkSuite"]["workflowRun"]["databaseId"]
+                workflow_ids.append(workflow_id)
+
+        workflow_ids = list(set(workflow_ids))
+        logging.info(f"Rerunning GitHub Actions workflows with IDs: {workflow_ids}")
+        if self.dry_run:
+            actions_github = None
+        else:
+            actions_github = GitHubRepo(
+                user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN
+            )
+        for workflow_id in workflow_ids:
+            if self.dry_run:
+                logging.info(f"Dry run, not restarting workflow {workflow_id}")
+            else:
+                try:
+                    actions_github.post(f"actions/runs/{workflow_id}/rerun-failed-jobs", data={})
+                except RuntimeError as e:
+                    logging.exception(e)
+                    # Ignore errors about jobs that are part of the same workflow to avoid
+                    # having to figure out which jobs are in which workflows ahead of time
+                    if "The workflow run containing this job is already running" in str(e):
+                        pass
+                    else:
+                        raise e
+
+    def comment_failure(self, msg: str, exceptions: Exception | list[Exception]):
+        if not isinstance(exceptions, list):
+            exceptions = [exceptions]
+
+        logging.info(f"Failed, commenting {exceptions}")
+
+        # Extract all the traceback strings
+        for item in exceptions:
+            try:
+                raise item
+            except Exception:
+                item.exception_msg = traceback.format_exc()
+
+        comment = f"{msg} in {args.run_url}\n\n"
+        for exception in exceptions:
+            comment += f"<details>\n\n```\n{exception.exception_msg}\n```\n\n"
+            if hasattr(exception, "read"):
+                comment += f"with response\n\n```\n{exception.read().decode()}\n```\n\n"
+            comment += "</details>"
+
+        pr.comment(comment)
+        pr.has_error = True
+        return exception
+
+
+def check_author(pr, triggering_comment, args):
+    comment_author = triggering_comment["user"]["login"]
+    if pr.author() == comment_author:
+        logging.info("Comment user is PR author, continuing")
+        return True
+    return False
+
+
+def search_users(name, triggering_comment, testing_json, search_fn):
+    logging.info(f"Checking {name}")
+    commment_author = triggering_comment["user"]["login"]
+    if testing_json:
+        matching_users = json.loads(testing_json)
+    else:
+        matching_users = search_fn(commment_author)
+    logging.info(f"Found {name}: {matching_users}")
+    user_names = {user["login"] for user in matching_users}
+
+    return len(matching_users) > 0 and commment_author in user_names
+
+
+def check_collaborator(pr, triggering_comment, args):
+    return search_users(
+        name="collaborators",
+        triggering_comment=triggering_comment,
+        search_fn=pr.search_collaborator,
+        testing_json=args.testing_collaborators_json,
+    )
+
+
+def check_mentionable_users(pr, triggering_comment, args):
+    return search_users(
+        name="mentionable users",
+        triggering_comment=triggering_comment,
+        search_fn=pr.search_mentionable_users,
+        testing_json=args.testing_mentionable_users_json,
+    )
+
+
+AUTH_CHECKS = {
+    "mentionable_users": check_mentionable_users,
+    "collaborators": check_collaborator,
+    "author": check_author,
+}
+# Stash the keys so they're accessible from the values
+AUTH_CHECKS = {k: (k, v) for k, v in AUTH_CHECKS.items()}
+
+
+class Merge:
+    triggers = [
+        "merge",
+        "merge this",
+        "merge this pr",
+    ]
+
+    auth = [AUTH_CHECKS["collaborators"], AUTH_CHECKS["author"]]
+
+    @staticmethod
+    def run(pr: PR):
+        info = None
+        try:
+            info = pr.merge_if_passed_checks()
+        except Exception as e:
+            pr.comment_failure("Failed to process merge request", e)
+            raise e
+
+        if info is not None:
+            try:
+                pr.trigger_gha_ci(sha=info["sha"])
+            except Exception as e:
+                pr.comment_failure("Failed to trigger GitHub Actions", e)
+                raise e
+
+
+class Rerun:
+    triggers = [
+        "rerun",
+        "rerun ci",
+        "re-run",
+        "re-run ci",
+        "run",
+        "run ci",
+    ]
+
+    auth = [AUTH_CHECKS["mentionable_users"]]
+
+    @staticmethod
+    def run(pr: PR):
+        errors = []
+        try:
+            pr.rerun_jenkins_ci()
+        except Exception as e:
+            logging.exception(e)
+            errors.append(e)
+
+        try:
+            pr.rerun_github_actions()
+        except Exception as e:
+            logging.exception(e)
+            errors.append(e)
+
+        if len(errors) > 0:
+            pr.comment_failure("Failed to re-run CI", errors)
+
+
+if __name__ == "__main__":
+    help = "Check if a PR has comments trying to merge it, and do so based on reviews/CI status"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--remote", default="origin", help="ssh remote to parse")
+    parser.add_argument("--pr", required=True, help="pr number to check")
+    parser.add_argument("--run-url", required=True, help="workflow run URL")
+    parser.add_argument(
+        "--trigger-comment-json", required=True, help="json of the comment that triggered this run"
+    )
+    parser.add_argument("--testing-pr-json", help="(testing only) manual data for testing")
+    parser.add_argument(
+        "--testing-collaborators-json", help="(testing only) manual data for testing"
+    )
+    parser.add_argument(
+        "--testing-mentionable-users-json", help="(testing only) manual data for testing"
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=False,
+        help="run but don't send any request to GitHub",
+    )
+    args = parser.parse_args()
+    init_log()
+    comment = json.loads(args.trigger_comment_json)
+    body = comment["body"].strip()
+
+    # Check that the comment was addressed to tvm-bot
+    if not body.startswith("@tvm-bot "):
+        logging.info(f"Not a bot comment, '{body}' does not start with '@tvm-bot'")
+        exit(0)
+
+    # Find the code to run for the command from the user
+    user_command = body.lstrip("@tvm-bot").strip()
+    command_to_run = None
+    for command in [Merge, Rerun]:
+        if user_command in command.triggers:
+            command_to_run = command
+            break
+
+    if command_to_run is None:
+        logging.info(f"Command '{user_command}' did not match anything")
+        exit(0)
+
+    # Find the remote for querying more data about the PR
+    remote = git(["config", "--get", f"remote.{args.remote}.url"])
+    logging.info(f"Using remote remote={remote}")
+    owner, repo = parse_remote(remote)
+
+    if args.pr.strip() == "":
+        logging.info("No PR number passed")
+        exit(0)
+
+    logging.info(f"Checking owner={owner} repo={repo}")
+    if args.testing_pr_json:
+        pr = PR(
+            number=int(args.pr),
+            owner=owner,
+            repo=repo,
+            dry_run=args.dry_run,
+            raw_data=json.loads(args.testing_pr_json),
+        )
+    else:
+        pr = PR(number=int(args.pr), owner=owner, repo=repo, dry_run=args.dry_run)
+
+    for name, check in command_to_run.auth:
+        if check(pr, comment, args):
+            logging.info(f"Passed auth check '{name}', continuing")
+            # Only one authorization check needs to pass (e.g. just mentionable
+            # or PR author), not all of them so quit
+            break
+        else:
+            logging.info(f"Failed auth check '{name}', quitting")
+            # Add a sad face
+            pr.react(comment, "confused")
+            exit(0)
+
+    # Acknowledge the comment with a react
+    pr.react(comment, "+1")
+
+    state = pr.state()
+
+    if state != "OPEN":
+        logging.info(f"Ignoring event on PR, state was not OPEN, instead was state={state}")
+        exit(0)
+
+    # Run the command
+    command_to_run.run(pr)
+
+    if pr.has_error:
+        raise RuntimeError("PR commented a failure")

ci/scripts/jenkins/cmd_utils.py

@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+assert (REPO_ROOT / "Jenkinsfile").exists
+
+
+class RelativePathFilter(logging.Filter):
+    def filter(self, record):
+        path = Path(record.pathname).resolve()
+        record.relativepath = str(path.relative_to(REPO_ROOT))
+        return True
+
+
+def init_log():
+    logging.basicConfig(
+        format="[%(relativepath)s:%(lineno)d %(levelname)-1s] %(message)s", level=logging.INFO
+    )
+
+    # Flush on every log call (logging and then calling subprocess.run can make
+    # the output look confusing)
+    logging.root.handlers[0].addFilter(RelativePathFilter())
+    logging.root.handlers[0].flush = sys.stderr.flush
+
+
+class Sh:
+    def __init__(self, env=None, cwd=None):
+        self.env = os.environ.copy()
+        if env is not None:
+            self.env.update(env)
+        self.cwd = cwd
+
+    def tee(self, cmd: str, **kwargs):
+        """
+        Run 'cmd' in a shell then return the (process, stdout) as a tuple
+        """
+
+        logging.info(f"+ {cmd}")
+
+        kwargs = {
+            **self._default_popen_flags(),
+            **kwargs,
+            "stdout": subprocess.PIPE,
+        }
+        proc = subprocess.Popen(cmd, **kwargs)
+
+        stdout = []
+
+        def _tee_output(s):
+            stdout.append(s)
+            print(s, end="")
+
+        while proc.poll() is None:
+            _tee_output(proc.stdout.readline())
+        _tee_output(proc.stdout.read())
+
+        stdout = "".join(stdout)
+        if proc.returncode:
+            raise subprocess.CalledProcessError(proc.returncode, proc.args, stdout)
+
+        return proc, stdout
+
+    def run(self, cmd: str, **kwargs):
+        logging.info(f"+ {cmd}")
+
+        kwargs = {
+            **self._default_popen_flags(),
+            "check": True,
+            **kwargs,
+        }
+
+        return subprocess.run(cmd, **kwargs)
+
+    def _default_popen_flags(self):
+        return {
+            "shell": True,
+            "env": self.env,
+            "encoding": "utf-8",
+            "cwd": self.cwd,
+        }
+
+
+def tags_from_title(title: str) -> list[str]:
+    tags = re.findall(r"\[(.*?)\]", title)
+    tags = [t.strip() for t in tags]
+    return tags

ci/scripts/jenkins/determine_docker_images.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E501
+import argparse
+import configparser
+import datetime
+import json
+import logging
+import urllib.error
+from pathlib import Path
+from typing import Any
+
+from cmd_utils import REPO_ROOT, init_log
+from http_utils import get
+
+DOCKER_API_BASE = "https://hub.docker.com/v2/"
+PAGE_SIZE = 25
+TEST_DATA = None
+IMAGE_TAGS_FILE = REPO_ROOT / "ci" / "jenkins" / "docker-images.ini"
+TVM_CI_ECR = "477529581014.dkr.ecr.us-west-2.amazonaws.com"
+
+
+def docker_api(url: str, use_pagination: bool = False) -> dict[str, Any]:
+    """
+    Run a paginated fetch from the public Docker Hub API
+    """
+    if TEST_DATA is not None:
+        if url not in TEST_DATA:
+            raise urllib.error.HTTPError(url, 404, "Not found", {}, None)
+        return TEST_DATA[url]
+    pagination = ""
+    if use_pagination:
+        pagination = f"?page_size={PAGE_SIZE}&page=1"
+    url = DOCKER_API_BASE + url + pagination
+    r, headers = get(url)
+    reset = headers.get("x-ratelimit-reset")
+    if reset is not None:
+        reset = datetime.datetime.fromtimestamp(int(reset))
+        reset = reset.isoformat()
+    logging.info(
+        f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})"
+    )
+    return r
+
+
+def image_exists(spec: str) -> bool:
+    name, tag = spec.split(":")
+    try:
+        r = docker_api(f"repositories/{name}/tags/{tag}")
+        logging.debug(f"Image exists, got response: {json.dumps(r, indent=2)}")
+        return True
+    except urllib.error.HTTPError as e:
+        # Image was not found
+        logging.debug(e)
+        return False
+
+
+def lookup_image_tag(name: str) -> str:
+    """Resolve image ``name`` (e.g. ``ci_cpu``) to its tag string from the ini.
+
+    Pure ini read — no Docker Hub query, no tlcpackstaging fallback. Used by
+    ``docker/dev_common.sh`` for local-dev image-name shortcuts where the
+    full Hub-existence check is unnecessary.
+    """
+    config = configparser.ConfigParser()
+    config.read(IMAGE_TAGS_FILE)
+    return config.get("jenkins", name)
+
+
+if __name__ == "__main__":
+    init_log()
+    parser = argparse.ArgumentParser(
+        description="Writes out Docker images names to be used to .docker-image-names/"
+    )
+    parser.add_argument(
+        "--lookup-only",
+        metavar="NAME",
+        help="Print the tag for NAME from the ini and exit (no Docker Hub query, no fallback).",
+    )
+    parser.add_argument(
+        "--testing-docker-data",
+        help="(testing only) JSON data to mock response from Docker Hub API",
+    )
+    parser.add_argument(
+        "--testing-images-data",
+        help=f"(testing only) JSON data to mock contents of {IMAGE_TAGS_FILE}",
+    )
+    parser.add_argument(
+        "--base-dir",
+        default=".docker-image-names",
+        help="(testing only) Folder to write image names to",
+    )
+    args, other = parser.parse_known_args()
+
+    if args.lookup_only:
+        print(lookup_image_tag(args.lookup_only))
+        raise SystemExit(0)
+
+    name_dir = Path(args.base_dir)
+
+    if args.testing_images_data:
+        repo_image_tags = json.loads(args.testing_images_data)
+    else:
+        config = configparser.ConfigParser()
+        config.read(IMAGE_TAGS_FILE)
+        repo_image_tags = {}
+        for name in other:
+            repo_image_tags[name] = config.get("jenkins", name)
+
+    images = {}
+    for name in other:
+        images[name] = repo_image_tags[name]
+
+    if args.testing_docker_data is not None:
+        TEST_DATA = json.loads(args.testing_docker_data)
+
+    logging.info(f"Checking if these images exist in tlcpack: {images}")
+
+    name_dir.mkdir(exist_ok=True)
+    images_to_use = {}
+    for filename, spec in images.items():
+        if spec.startswith(TVM_CI_ECR):
+            logging.info(f"{spec} is from ECR")
+            images_to_use[filename] = spec
+        elif image_exists(spec):
+            logging.info(f"{spec} found in tlcpack")
+            images_to_use[filename] = spec
+        else:
+            logging.info(f"{spec} not found in tlcpack, using tlcpackstaging")
+            part, tag = spec.split(":")
+            user, repo = part.split("/")
+            tlcpackstaging_tag = f"tlcpackstaging/{repo.replace('-', '_')}:{tag}"
+            images_to_use[filename] = tlcpackstaging_tag
+
+    for filename, image in images_to_use.items():
+        logging.info(f"Writing image {image} to {name_dir / filename}")
+        with open(name_dir / filename, "w") as f:
+            f.write(image)

ci/scripts/jenkins/git_change_docker.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+if [ "${BRANCH_NAME}" == "main" ]; then
+    changed_files=$(git diff --no-commit-id --name-only -r HEAD~1)
+else
+    changed_files=$(git diff --no-commit-id --name-only -r origin/main)
+fi
+
+FILES_THAT_SHOULDNT_TRIGGER_REBUILDS=(
+    "docker/bash.sh"
+    "docker/with_the_same_user"
+    "docker/README.md"
+    "docker/Dockerfile.ci_lint"
+    "docker/install/ubuntu_install_clang_format.sh"
+    "docker/lint.sh"
+    "docker/clear-stale-images.sh"
+    "README.md"
+    "lint.sh"
+    "clear-stale-images.sh"
+)
+
+for file in $changed_files; do
+    # Certain files under docker/ don't matter for rebuilds, so ignore them
+    if printf '%s\0' "${FILES_THAT_SHOULDNT_TRIGGER_REBUILDS[@]}" | grep -F -x -z -- "$file"; then
+        echo "Skipping $file"
+        continue
+    fi
+    # if grep -q "docker/"
+    echo "Checking $file"
+    if grep -q "docker/" <<< "$file"; then
+        exit 1
+    fi
+done
+
+# No docker changes
+exit 0

ci/scripts/jenkins/git_change_docs.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+FOUND_ONE_FILE=0
+SAW_NON_DOC_CHANGES=0
+
+changed_files=$(git diff --no-commit-id --name-only -r origin/main)
+
+for file in $changed_files; do
+    FOUND_ONE_FILE=1
+    if ! grep -q "docs/" <<< "$file"; then
+        SAW_NON_DOC_CHANGES=1
+        break
+    fi
+done
+
+if [ ${FOUND_ONE_FILE} -eq 0 ] || [ ${SAW_NON_DOC_CHANGES} -eq 1 ]; then
+    exit 0
+else
+    exit 1
+fi

ci/scripts/jenkins/git_skip_ci.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+import os
+
+from cmd_utils import init_log, tags_from_title
+from git_utils import GitHubRepo, git, parse_remote
+
+if __name__ == "__main__":
+    help = "Exits with 0 if CI should be skipped, 1 otherwise"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--pr", required=True)
+    parser.add_argument("--remote", default="origin", help="ssh remote to parse")
+    parser.add_argument(
+        "--pr-title", help="(testing) PR title to use instead of fetching from GitHub"
+    )
+    args = parser.parse_args()
+    init_log()
+
+    branch = git(["rev-parse", "--abbrev-ref", "HEAD"])
+    log = git(["log", "--format=%s", "-1"])
+
+    # Check the PR's title (don't check this until everything else passes first)
+    def check_pr_title():
+        remote = git(["config", "--get", f"remote.{args.remote}.url"])
+        user, repo = parse_remote(remote)
+
+        if args.pr_title:
+            title = args.pr_title
+        else:
+            github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo)
+            pr = github.get(f"pulls/{args.pr}")
+            title = pr["title"]
+        logging.info(f"pr title: {title}")
+        tags = tags_from_title(title)
+        logging.info(f"Found title tags: {tags}")
+        return "skip ci" in tags
+
+    if args.pr != "null" and args.pr.strip() != "" and branch != "main" and check_pr_title():
+        logging.info("PR title starts with '[skip ci]', skipping...")
+        exit(0)
+    else:
+        logging.info(f"Not skipping CI:\nargs.pr: {args.pr}\nbranch: {branch}\ncommit: {log}")
+        exit(1)

ci/scripts/jenkins/git_skip_ci_globs.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E501
+
+import argparse
+import fnmatch
+
+from git_utils import git
+
+globs = [
+    "*.md",
+    ".github/*",
+    ".asf.yaml",
+    ".gitignore",
+    "LICENSE",
+    "NOTICE",
+    "KEYS",
+    "tests/lint/*",
+    ".pre-commit-config.yaml",
+]
+
+
+def match_any(f: str) -> str | None:
+    for glob in globs:
+        if fnmatch.fnmatch(f, glob):
+            return glob
+    return None
+
+
+if __name__ == "__main__":
+    help = "Exits with code 1 if a change only touched files, indicating that CI could be skipped for this changeset"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--files", help="(testing only) comma separated list of files to check")
+    args = parser.parse_args()
+    print(args)
+    if args.files is not None:
+        diff = [x for x in args.files.split(",") if x.strip() != ""]
+    else:
+        diff = git(["diff", "--no-commit-id", "--name-only", "-r", "origin/main"])
+        diff = diff.split("\n")
+        diff = [d.strip() for d in diff]
+        diff = [d for d in diff if d != ""]
+
+    print(f"Changed files:\n{diff}")
+
+    if len(diff) == 0:
+        print("Found no changed files, skipping CI")
+        exit(0)
+
+    print(f"Checking with globs:\n{globs}")
+
+    for file in diff:
+        match = match_any(file)
+        if match is None:
+            print(f"{file} did not match any globs, running CI")
+            exit(1)
+        else:
+            print(f"{file} matched glob {match}")
+
+    print("All files matched a glob, skipping CI")
+    exit(0)

ci/scripts/jenkins/git_utils.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E501, RUF005
+
+import base64
+import json
+import logging
+import os
+import re
+import subprocess
+from typing import Any
+from urllib import error, request
+
+DRY_RUN = object()
+
+
+def compress_query(query: str) -> str:
+    query = query.replace("\n", "")
+    query = re.sub("\s+", " ", query)
+    return query
+
+
+def post(url: str, body: Any | None = None, auth: tuple[str, str] | None = None):
+    logging.info(f"Requesting POST to {url} with {body}")
+    headers = {}
+    req = request.Request(url, headers=headers, method="POST")
+    if auth is not None:
+        auth_str = base64.b64encode(f"{auth[0]}:{auth[1]}".encode())
+        req.add_header("Authorization", f"Basic {auth_str.decode()}")
+
+    if body is None:
+        body = ""
+
+    req.add_header("Content-Type", "application/json; charset=utf-8")
+    data = json.dumps(body)
+    data = data.encode("utf-8")
+    req.add_header("Content-Length", len(data))
+
+    with request.urlopen(req, data) as response:
+        return response.read()
+
+
+def dry_run_token(is_dry_run: bool) -> Any:
+    if is_dry_run:
+        return DRY_RUN
+    return os.environ["GITHUB_TOKEN"]
+
+
+class GitHubRepo:
+    GRAPHQL_URL = "https://api.github.com/graphql"
+
+    def __init__(self, user, repo, token, test_data=None):
+        self.token = token
+        self.user = user
+        self.repo = repo
+        self.test_data = test_data
+        self.num_calls = 0
+        self.base = f"https://api.github.com/repos/{user}/{repo}/"
+
+    def headers(self):
+        return {
+            "Authorization": f"Bearer {self.token}",
+        }
+
+    def dry_run(self) -> bool:
+        return self.token == DRY_RUN
+
+    def graphql(self, query: str, variables: dict[str, str] | None = None) -> dict[str, Any]:
+        query = compress_query(query)
+        if variables is None:
+            variables = {}
+
+        response = self._request(
+            self.GRAPHQL_URL,
+            {"query": query, "variables": variables},
+            method="POST",
+        )
+        if self.dry_run():
+            return self.testing_response("POST", self.GRAPHQL_URL)
+
+        if "data" not in response:
+            msg = f"Error fetching data with query:\n{query}\n\nvariables:\n{variables}\n\nerror:\n{json.dumps(response, indent=2)}"
+            raise RuntimeError(msg)
+        return response
+
+    def testing_response(self, method: str, url: str) -> Any:
+        self.num_calls += 1
+        key = f"[{self.num_calls}] {method} - {url}"
+        if self.test_data is not None and key in self.test_data:
+            return self.test_data[key]
+        logging.info(f"Unknown URL in dry run: {key}")
+        return {}
+
+    def _request(self, full_url: str, body: dict[str, Any], method: str) -> dict[str, Any]:
+        if self.dry_run():
+            logging.info(f"Dry run, would have requested a {method} to {full_url} with {body}")
+            return self.testing_response(method, full_url)
+
+        logging.info(f"Requesting {method} to {full_url} with {body}")
+        req = request.Request(full_url, headers=self.headers(), method=method.upper())
+        req.add_header("Content-Type", "application/json; charset=utf-8")
+        data = json.dumps(body)
+        data = data.encode("utf-8")
+        req.add_header("Content-Length", len(data))
+
+        try:
+            with request.urlopen(req, data) as response:
+                content = response.read()
+        except error.HTTPError as e:
+            msg = str(e)
+            error_data = e.read().decode()
+            raise RuntimeError(f"Error response: {msg}\n{error_data}")
+
+        logging.info(f"Got response from {full_url}: {content}")
+        try:
+            response = json.loads(content)
+        except json.decoder.JSONDecodeError:
+            return content
+
+        return response
+
+    def put(self, url: str, data: dict[str, Any]) -> dict[str, Any]:
+        return self._request(self.base + url, data, method="PUT")
+
+    def patch(self, url: str, data: dict[str, Any]) -> dict[str, Any]:
+        return self._request(self.base + url, data, method="PATCH")
+
+    def post(self, url: str, data: dict[str, Any]) -> dict[str, Any]:
+        return self._request(self.base + url, data, method="POST")
+
+    def get(self, url: str) -> dict[str, Any]:
+        if self.dry_run():
+            logging.info(f"Dry run, would have requested a GET to {url}")
+            return self.testing_response("GET", url)
+        url = self.base + url
+        logging.info(f"Requesting GET to {url}")
+        req = request.Request(url, headers=self.headers())
+        with request.urlopen(req) as response:
+            response = json.loads(response.read())
+        return response
+
+    def delete(self, url: str) -> dict[str, Any]:
+        if self.dry_run():
+            logging.info(f"Dry run, would have requested a DELETE to {url}")
+            return self.testing_response("DELETE", url)
+        url = self.base + url
+        logging.info(f"Requesting DELETE to {url}")
+        req = request.Request(url, headers=self.headers(), method="DELETE")
+        with request.urlopen(req) as response:
+            response = json.loads(response.read())
+        return response
+
+
+def parse_remote(remote: str) -> tuple[str, str]:
+    """
+    Get a GitHub (user, repo) pair out of a git remote
+    """
+    if remote.startswith("https://"):
+        # Parse HTTP remote
+        parts = remote.split("/")
+        if len(parts) < 2:
+            raise RuntimeError(f"Unable to parse remote '{remote}'")
+        user, repo = parts[-2], parts[-1].replace(".git", "")
+    else:
+        # Parse SSH remote
+        m = re.search(r":(.*)/(.*)\.git", remote)
+        if m is None or len(m.groups()) != 2:
+            raise RuntimeError(f"Unable to parse remote '{remote}'")
+        user, repo = m.groups()
+
+    user = os.getenv("DEBUG_USER", user)
+    repo = os.getenv("DEBUG_REPO", repo)
+    return user, repo
+
+
+def git(command, **kwargs):
+    command = ["git"] + command
+    logging.info(f"Running {command}")
+    proc = subprocess.run(command, stdout=subprocess.PIPE, encoding="utf-8", **kwargs)
+    if proc.returncode != 0:
+        raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}")
+    return proc.stdout.strip()
+
+
+def find_ccs(body: str) -> list[str]:
+    matches = re.findall(r"(cc( @[-A-Za-z0-9]+)+)", body, flags=re.MULTILINE)
+    matches = [full for full, last in matches]
+
+    reviewers = []
+    for match in matches:
+        if match.startswith("cc "):
+            match = match.replace("cc ", "")
+        users = [x.strip() for x in match.split("@")]
+        reviewers += users
+
+    reviewers = set(x for x in reviewers if x != "")
+    return list(reviewers)

ci/scripts/jenkins/http_utils.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import logging
+from typing import Any
+from urllib import request
+
+
+def get(url: str, headers: dict[str, str] | None = None) -> dict[str, Any]:
+    logging.info(f"Requesting GET to {url}")
+    if headers is None:
+        headers = {}
+    req = request.Request(url, headers=headers)
+    with request.urlopen(req) as response:
+        response_headers = {k: v for k, v in response.getheaders()}
+        response = json.loads(response.read())
+
+    return response, response_headers

ci/scripts/jenkins/open_docker_update_pr.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import configparser
+import datetime
+import json
+import logging
+import os
+import re
+import shlex
+from collections.abc import Callable
+from typing import Any
+from urllib import error
+
+from cmd_utils import REPO_ROOT, Sh, init_log
+from git_utils import GitHubRepo, git, parse_remote
+from should_rebuild_docker import docker_api
+
+JENKINS_DIR = REPO_ROOT / "ci" / "jenkins"
+IMAGES_FILE = JENKINS_DIR / "docker-images.ini"
+GENERATE_SCRIPT = JENKINS_DIR / "generate.py"
+GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
+BRANCH = "nightly-docker-update"
+
+
+def _testing_docker_api(data: dict[str, Any]) -> Callable[[str], dict[str, Any]]:
+    """Returns a function that can be used in place of docker_api"""
+
+    def mock(url: str) -> dict[str, Any]:
+        if url in data:
+            return data[url]
+        else:
+            raise error.HTTPError(url, 404, f"Not found: {url}", {}, None)
+
+    return mock
+
+
+def parse_docker_date(d: str) -> datetime.datetime:
+    """Turn a date string from the Docker API into a datetime object"""
+    return datetime.datetime.strptime(d, "%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def check_tag(tag: dict[str, Any]) -> bool:
+    return re.match(r"^[0-9]+-[0-9]+-[a-z0-9]+$", tag["name"]) is not None
+
+
+def latest_tag(user: str, repo: str) -> list[dict[str, Any]]:
+    """
+    Queries Docker Hub and finds the most recent tag for the specified image/repo pair
+    """
+    r = docker_api(f"repositories/{user}/{repo}/tags")
+    results = r["results"]
+
+    for result in results:
+        result["last_updated"] = parse_docker_date(result["last_updated"])
+
+    results = list(sorted(results, key=lambda d: d["last_updated"]))
+    results = [tag for tag in results if check_tag(tag)]
+    return results[-1]
+
+
+def latest_tlcpackstaging_image(source: str) -> str | None:
+    """
+    Finds the latest full tag to use in the Jenkinsfile or returns None if no
+    update is needed
+    """
+    name, current_tag = source.split(":")
+    user, repo = name.split("/")
+    logging.info(
+        f"Running with name: {name}, current_tag: {current_tag}, user: {user}, repo: {repo}"
+    )
+
+    staging_repo = repo.replace("-", "_")
+    latest_tlcpackstaging_tag = latest_tag(user="tlcpackstaging", repo=staging_repo)
+    logging.info(f"Found latest tlcpackstaging tag:\n{latest_tlcpackstaging_tag}")
+
+    if latest_tlcpackstaging_tag["name"] == current_tag:
+        logging.info("tlcpackstaging tag is the same as the one in the Jenkinsfile")
+
+    latest_tlcpack_tag = latest_tag(user="tlcpack", repo=repo)
+    logging.info(f"Found latest tlcpack tag:\n{latest_tlcpack_tag}")
+
+    if latest_tlcpack_tag["name"] == latest_tlcpackstaging_tag["name"]:
+        logging.info("Tag names were the same, no update needed")
+        return None
+
+    if latest_tlcpack_tag["last_updated"] > latest_tlcpackstaging_tag["last_updated"]:
+        new_spec = f"tlcpack/{repo}:{latest_tlcpack_tag['name']}"
+    else:
+        # Even if the image doesn't exist in tlcpack, it will fall back to tlcpackstaging
+        # so hardcode the username here
+        new_spec = f"tlcpack/{repo}:{latest_tlcpackstaging_tag['name']}"
+        logging.info("Using tlcpackstaging tag on tlcpack")
+
+    logging.info(f"Found newer image, using: {new_spec}")
+    return new_spec
+
+
+if __name__ == "__main__":
+    init_log()
+    help = "Open a PR to update the Docker images to use the latest available in tlcpackstaging"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--remote", default="origin", help="ssh remote to parse")
+    parser.add_argument("--dry-run", action="store_true", help="don't send PR to GitHub")
+    parser.add_argument("--testing-docker-data", help="JSON data to mock Docker Hub API response")
+    args = parser.parse_args()
+
+    # Install test mock if necessary
+    if args.testing_docker_data is not None:
+        docker_api = _testing_docker_api(data=json.loads(args.testing_docker_data))
+
+    remote = git(["config", "--get", f"remote.{args.remote}.url"])
+    user, repo = parse_remote(remote)
+
+    # Read the existing images from ci/jenkins/docker-images.ini.
+    # The ini has a single ``[jenkins]`` section with a shared ``ci_tag`` key
+    # and one ``ci_<name>: tlcpack/ci-<name>:%(ci_tag)s`` entry per image.
+    # Resolve each image to its full spec (with interpolation applied) and
+    # check against Docker Hub for newer tags.
+    logging.info(f"Reading {IMAGES_FILE}")
+    config = configparser.ConfigParser()
+    config.read(IMAGES_FILE)
+    with open(IMAGES_FILE) as f:
+        content = f.read()
+
+    replacements = {}
+    for key in config.options("jenkins"):
+        if key == "ci_tag":
+            continue
+        image_spec = config.get("jenkins", key)
+        logging.info(f"Found {key} = {image_spec}")
+        new_image = latest_tlcpackstaging_image(image_spec)
+        if new_image is None:
+            logging.info("No new image found")
+            continue
+        logging.info(f"Using new image {new_image}")
+        # Rewrite the ``ci_<name>:`` line with the resolved tag (breaking
+        # the ``%(ci_tag)s`` interpolation for that single entry) so the
+        # update is unambiguous and doesn't disturb other images that share
+        # the old tag.
+        old_line_re = re.compile(rf"^{re.escape(key)}\s*[:=].*$", re.MULTILINE)
+        new_line = f"{key}: {new_image}"
+        replacements[old_line_re] = new_line
+
+    # Re-generate the Jenkinsfiles
+    command = f"python3 {shlex.quote(str(GENERATE_SCRIPT))}"
+
+    for old_line_re, new_line in replacements.items():
+        content = old_line_re.sub(new_line, content)
+
+    print(f"Updated to:\n{content}")
+
+    if args.dry_run:
+        print(f"Would have run:\n{command}")
+    else:
+        with open(IMAGES_FILE, "w") as f:
+            f.write(content)
+        Sh().run(command)
+
+    # Publish the PR
+    title = "[ci][docker] Nightly Docker image update"
+    body = "This bumps the Docker images to the latest versions from Docker Hub."
+    message = f"{title}\n\n\n{body}"
+
+    if args.dry_run:
+        logging.info("Dry run, would have committed Jenkinsfiles")
+    else:
+        logging.info("Creating git commit")
+        git(["checkout", "-B", BRANCH])
+        git(["add", str(JENKINS_DIR.relative_to(REPO_ROOT))])
+        git(["config", "user.name", "tvm-bot"])
+        git(["config", "user.email", "95660001+tvm-bot@users.noreply.github.com"])
+        git(["commit", "-m", message])
+        git(["push", "--set-upstream", args.remote, BRANCH, "--force"])
+
+    logging.info("Sending PR to GitHub")
+    github = GitHubRepo(user=user, repo=repo, token=GITHUB_TOKEN)
+    data = {
+        "title": title,
+        "body": body,
+        "head": BRANCH,
+        "base": "main",
+        "maintainer_can_modify": True,
+    }
+    url = "pulls"
+    if args.dry_run:
+        logging.info(f"Dry run, would have sent {data} to {url}")
+    else:
+        try:
+            github.post(url, data=data)
+        except error.HTTPError as e:
+            # Ignore the exception if the PR already exists (which gives a 422). The
+            # existing PR will have been updated in place
+            if e.code == 422:
+                logging.info("PR already exists, ignoring error")
+                logging.exception(e)
+            else:
+                raise e

ci/scripts/jenkins/retry.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+retry() {
+  local max_retries=$1
+  shift
+  local n=0
+  until [ "$n" -ge "$max_retries" ]
+  do
+      "$@" && break
+      n=$((n+1))
+      if [ "$n" -eq "$max_retries" ]; then
+          echo "failed to update after attempt $n / $max_retries, giving up"
+          exit 1
+      fi
+
+      WAIT=$(python3 -c 'import random; print(random.randint(30, 200))')
+      echo "failed to update $n / $max_retries, waiting $WAIT to try again"
+      sleep "$WAIT"
+  done
+}

ci/scripts/jenkins/s3.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import importlib.util
+import logging
+import re
+from enum import Enum
+from pathlib import Path
+
+from cmd_utils import REPO_ROOT, Sh, init_log
+
+DATA_PY = REPO_ROOT / "ci" / "jenkins" / "data.py"
+
+
+def load_files_to_stash():
+    """Load the files_to_stash dict from ci/jenkins/data.py."""
+    spec = importlib.util.spec_from_file_location("data", DATA_PY)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod.files_to_stash
+
+
+def resolve_bundles(bundle_names: list[str]) -> list[str]:
+    """Resolve a list of bundle names to a flat list of file paths."""
+    files_to_stash = load_files_to_stash()
+    items = []
+    for name in bundle_names:
+        if name not in files_to_stash:
+            known = list(files_to_stash.keys())
+            logging.error(f"Unknown bundle '{name}'. Known bundles: {known}")
+            raise SystemExit(1)
+        items.extend(files_to_stash[name])
+    return items
+
+
+RETRY_SCRIPT = REPO_ROOT / "ci" / "scripts" / "jenkins" / "retry.sh"
+S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)")
+SH = Sh()
+
+
+class Action(Enum):
+    UPLOAD = 1
+    DOWNLOAD = 2
+
+
+def show_md5(item: str) -> None:
+    if not Path(item).is_dir():
+        sh.run(f"md5sum {item}")
+
+
+def parse_output_files(stdout: str) -> list[str]:
+    """
+    Grab the list of downloaded files from the output of 'aws s3 cp'. Lines look
+    like:
+
+        download: s3://some/prefix/a_file.txt to a_file.txt
+    """
+    files = []
+    for line in stdout.split("\n"):
+        line = line.strip()
+        if line == "":
+            continue
+        m = S3_DOWNLOAD_REGEX.match(line)
+        if m:
+            files.append(m.groups()[0])
+
+    return files
+
+
+def chmod(files: list[str]) -> None:
+    """
+    S3 has no concept of file permissions so add them back in here to every file
+    """
+    # Add execute bit for downloads
+    to_chmod = [str(f) for f in files]
+    logging.info(f"Adding execute bit for files: {to_chmod}")
+    if len(to_chmod) > 0:
+        SH.run(f"chmod +x {' '.join(to_chmod)}")
+
+
+def s3(source: str, destination: str, recursive: bool) -> list[str]:
+    """
+    Send or download the source to the destination in S3
+    """
+    cmd = f". {RETRY_SCRIPT.relative_to(REPO_ROOT)} && retry 3 aws s3 cp --no-progress"
+
+    if recursive:
+        cmd += " --recursive"
+
+    cmd += f" {source} {destination}"
+    _, stdout = SH.tee(cmd)
+    return stdout
+
+
+if __name__ == "__main__":
+    init_log()
+    help = "Uploads or downloads files from S3"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--action", help="either 'upload' or 'download'", required=True)
+    parser.add_argument("--bucket", help="s3 bucket", required=True)
+    parser.add_argument(
+        "--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu", required=True
+    )
+    parser.add_argument("--items", help="files and folders to upload", nargs="+")
+    parser.add_argument(
+        "--bundle",
+        help="bundle name(s) from ci/jenkins/data.py files_to_stash (repeatable)",
+        action="append",
+        dest="bundles",
+        metavar="NAME",
+    )
+
+    args = parser.parse_args()
+
+    if args.items is not None and args.bundles is not None:
+        parser.error("--items and --bundle are mutually exclusive")
+
+    logging.info(args)
+
+    sh = Sh()
+
+    if Path.cwd() != REPO_ROOT:
+        logging.error(f"s3.py can only be executed from the repo root, instead was in {Path.cwd()}")
+        exit(1)
+
+    prefix = args.prefix.strip("/")
+    s3_path = f"s3://{args.bucket}/{prefix}"
+    logging.info(f"Using s3 path: {s3_path}")
+
+    if args.action == "upload":
+        action = Action.UPLOAD
+    elif args.action == "download":
+        action = Action.DOWNLOAD
+    else:
+        logging.error(f"Unsupported action: {args.action}")
+        exit(1)
+
+    if args.bundles is not None:
+        items = resolve_bundles(args.bundles)
+    elif args.items is not None:
+        items = args.items
+    else:
+        if args.action == "upload":
+            logging.error("Cannot upload without --items or --bundle")
+            exit(1)
+        else:
+            # Download the whole prefix
+            items = ["."]
+
+    for item in items:
+        if action == Action.DOWNLOAD:
+            source = s3_path
+            recursive = True
+            if item != ".":
+                source = s3_path + "/" + item
+                recursive = False
+            try:
+                stdout = s3(source=source, destination=item, recursive=recursive)
+            except Exception:
+                # Optional artifacts (e.g. per-backend device runtime DSOs) may not
+                # exist in S3 when the build config didn't produce them. Skip silently.
+                logging.warning(f"Download failed for {item}, skipping (may be optional)")
+                continue
+            files = parse_output_files(stdout)
+            chmod(files)
+            for file in files:
+                # Show md5 after downloading
+                show_md5(file)
+        elif action == Action.UPLOAD:
+            if not Path(item).exists():
+                logging.warning(f"The path doesn't exist: {item}")
+                continue
+            show_md5(item)
+            if Path(item).is_dir():
+                if len(list(Path(item).glob("**/*"))) == 0:
+                    raise RuntimeError(f"Cannot upload empty folder with name: {item}")
+            s3(item, s3_path + "/" + item, recursive=Path(item).is_dir())

ci/scripts/jenkins/should_rebuild_docker.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E501
+import argparse
+import datetime
+import json
+import logging
+import subprocess
+from typing import Any
+
+from cmd_utils import Sh, init_log
+from http_utils import get
+
+DOCKER_API_BASE = "https://hub.docker.com/v2/"
+PAGE_SIZE = 25
+TEST_DATA = None
+
+
+def docker_api(url: str) -> dict[str, Any]:
+    """
+    Run a paginated fetch from the public Docker Hub API
+    """
+    if TEST_DATA is not None:
+        return TEST_DATA[url]
+    pagination = f"?page_size={PAGE_SIZE}&page=1"
+    url = DOCKER_API_BASE + url + pagination
+    r, headers = get(url)
+    reset = headers.get("x-ratelimit-reset")
+    if reset is not None:
+        reset = datetime.datetime.fromtimestamp(int(reset))
+        reset = reset.isoformat()
+    logging.info(
+        f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})"
+    )
+    if "results" not in r:
+        raise RuntimeError(f"Error fetching data, no results found in: {r}")
+    return r
+
+
+def any_docker_changes_since(hash: str) -> bool:
+    """
+    Check the docker/ directory, return True if there have been any code changes
+    since the specified hash
+    """
+    sh = Sh()
+    cmd = f"git diff {hash} -- docker/"
+    proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    stdout = proc.stdout.strip()
+    return stdout != "", stdout
+
+
+def does_commit_exist(hash: str) -> bool:
+    """
+    Returns True if the hash exists in the repo
+    """
+    sh = Sh()
+    cmd = f"git rev-parse -q {hash}"
+    proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
+    print(proc.stdout)
+    if proc.returncode == 0:
+        return True
+
+    if "unknown revision or path not in the working tree" in proc.stdout:
+        return False
+
+    raise RuntimeError(f"Unexpected failure when running: {cmd}")
+
+
+def find_hash_for_tag(tag: dict[str, Any]) -> str:
+    """
+    Split the hash off of a name like <date>-<time>-<hash>
+    """
+    name = tag["name"]
+    name_parts = name.split("-")
+    if len(name_parts) != 3:
+        raise RuntimeError(f"Image {name} is not using new naming scheme")
+    shorthash = name_parts[2]
+    return shorthash
+
+
+def find_commit_in_repo(tags: list[dict[str, Any]]):
+    """
+    Look through all the docker tags, find the most recent one which references
+    a commit that is present in the repo
+    """
+    for tag in tags["results"]:
+        shorthash = find_hash_for_tag(tag)
+        logging.info(f"Hash '{shorthash}' does not exist in repo")
+        if does_commit_exist(shorthash):
+            return shorthash, tag
+
+    raise RuntimeError(f"No extant hash found in tags:\n{tags}")
+
+
+def main():
+    # Fetch all tlcpack images
+    images = docker_api("repositories/tlcpack")
+
+    # Ignore all non-ci images
+    relevant_images = [image for image in images["results"] if image["name"].startswith("ci-")]
+    image_names = [image["name"] for image in relevant_images]
+    logging.info(f"Found {len(relevant_images)} images to check: {', '.join(image_names)}")
+
+    for image in relevant_images:
+        # Check the tags for the image
+        tags = docker_api(f"repositories/tlcpack/{image['name']}/tags")
+
+        # Find the hash of the most recent tag
+        shorthash, tag = find_commit_in_repo(tags)
+        name = tag["name"]
+        logging.info(f"Looking for docker/ changes since {shorthash}")
+
+        any_docker_changes, diff = any_docker_changes_since(shorthash)
+        if any_docker_changes:
+            logging.info(f"Found docker changes from {shorthash} when checking {name}")
+            logging.info(diff)
+            exit(2)
+
+    logging.info("Did not find changes, no rebuild necessary")
+    exit(0)
+
+
+if __name__ == "__main__":
+    init_log()
+    parser = argparse.ArgumentParser(
+        description="Exits 0 if Docker images don't need to be rebuilt, 1 otherwise"
+    )
+    parser.add_argument(
+        "--testing-docker-data",
+        help="(testing only) JSON data to mock response from Docker Hub API",
+    )
+    args = parser.parse_args()
+
+    if args.testing_docker_data is not None:
+        TEST_DATA = json.loads(args.testing_docker_data)
+
+    main()

ci/scripts/jenkins/should_run_slow_tests.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import textwrap
+
+from git_utils import GitHubRepo, git, parse_remote
+
+SLOW_TEST_TRIGGERS = [
+    "@tvm-bot run slow tests",
+    "@tvm-bot run slow test",
+    "@tvm-bot run slow",
+    "@tvm-bot slow tests",
+    "@tvm-bot slow test",
+    "@tvm-bot slow",
+]
+
+
+def check_match(s: str, searches: list[str]) -> tuple[bool, str | None]:
+    for search in searches:
+        if search in s:
+            return True, search
+
+    return False, None
+
+
+def display(long_str: str) -> str:
+    return textwrap.indent(long_str, "    ")
+
+
+if __name__ == "__main__":
+    help = "Exits with 1 if CI should run slow tests, 0 otherwise"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--pr", required=True)
+    parser.add_argument("--remote", default="origin", help="ssh remote to parse")
+    parser.add_argument(
+        "--pr-body", help="(testing) PR body to use instead of fetching from GitHub"
+    )
+    args = parser.parse_args()
+
+    branch = git(["rev-parse", "--abbrev-ref", "HEAD"])
+
+    # Don't skip slow tests on main or ci-docker-staging
+    skip_branches = {"main", "ci-docker-staging"}
+    if branch in skip_branches:
+        print(f"Branch {branch} is in {skip_branches}, running slow tests")
+        exit(1)
+    print(f"Branch {branch} is not in {skip_branches}, checking last commit...")
+
+    if args.pr_body:
+        body = args.pr_body
+    else:
+        remote = git(["config", "--get", f"remote.{args.remote}.url"])
+        user, repo = parse_remote(remote)
+
+        github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo)
+        pr = github.get(f"pulls/{args.pr}")
+        body = pr["body"]
+
+    body_match, reason = check_match(body, SLOW_TEST_TRIGGERS)
+
+    if body_match:
+        print(f"Matched {reason} in PR body:\n{display(body)}, running slow tests")
+        exit(1)
+
+    print(
+        f"PR Body:\n{display(body)}\ndid not have any of {SLOW_TEST_TRIGGERS}, skipping slow tests"
+    )
+    exit(0)

ci/scripts/package/README.md

@@ -0,0 +1,31 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# TVM wheel packaging
+
+The wheels are built by a standard `cibuildwheel` flow, configured in
+`.github/workflows/publish_wheel.yml` and `pyproject.toml` (`[tool.cibuildwheel]`
+and `[tool.scikit-build]`). This directory holds the few helper scripts that flow
+invokes:
+
+- `manylinux_build_libtvm_runtime_cuda.sh` — run by the `build_cuda_runtime` CI
+  stage; builds the `libtvm_runtime_cuda.so` sidecar inside the prebuilt
+  `quay.io/manylinux_cuda` image (CUDA toolkit preinstalled).
+- `windows_build_libtvm_runtime_cuda.bat` — the Windows equivalent (run with
+  `shell: cmd`); installs the CUDA toolkit via conda and builds
+  `tvm_runtime_cuda.dll`.
+- `build-environment.yaml` — conda environment for building the wheel.

ci/scripts/package/build-environment.yaml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the specific
+# language governing permissions and limitations under the License.
+
+# Build environment for TVM wheel building.
+# This environment provides the necessary dependencies for building TVM wheels.
+name: tvm-wheel-build
+
+# The conda channels to lookup the dependencies
+channels:
+  - conda-forge
+
+# The packages to install to the environment
+dependencies:
+  # Core build tools
+  - cmake >=3.24
+  - ninja
+  - make
+  - llvmdev >=11
+  - python >=3.10
+  - pip
+  - git
+  - bzip2
+  - zlib
+  - zstd-static
+  - pytest
+  - numpy
+  - scipy
+  - cython
+  - libxml2-devel

ci/scripts/package/manylinux_build_libtvm_runtime_cuda.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Build libtvm_runtime_cuda.so inside a manylinux CUDA container, run by the
+# build_cuda_runtime CI job. The official quay.io/manylinux_cuda images ship
+# the CUDA toolkit preinstalled under /usr/local/cuda, but omit the libcuda
+# driver stub, so we install just cuda-driver-devel from the image's CUDA repo.
+# Builds the sidecar into build-wheel-cuda/lib/ for the wheel build to bundle.
+#
+# Usage: manylinux_build_libtvm_runtime_cuda.sh
+set -euxo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+build_dir="${repo_root}/build-wheel-cuda"
+parallel="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)"
+
+# The image ships the toolkit but not the libcuda driver stub; install it from
+# the image's CUDA repo so the sidecar links libcuda.so.1 (sync version with tag).
+dnf -y install cuda-driver-devel-13-1
+
+# Build the CUDA runtime sidecar with CUDA on and LLVM off, so it does not need
+# the LLVM prefix; the main CPU wheel links LLVM statically. The manylinux CUDA
+# image already ships cmake and make, and the build uses the default Makefiles
+# generator (no Ninja), so no build tools are installed here. Put the bundled
+# CPython and CUDA toolchain on PATH for the CMake configure and nvcc.
+export PATH="/opt/python/cp310-cp310/bin:/usr/local/cuda/bin:${PATH}"
+nvcc --version
+
+rm -rf "${build_dir}"
+# CMAKE_CUDA_COMPILER only tells CMake which nvcc to use; it does not affect the
+# resulting libtvm_runtime_cuda.so, which is built only from .cc host sources (no
+# .cu device code, so nvcc is never invoked for it). CMAKE_CUDA_ARCHITECTURES is
+# intentionally not set: it would be a no-op here for the same reason (verified --
+# the .so is byte-identical across arch values and carries no device code), and
+# modern CMake fills in a default so configure does not fail without it.
+cmake -S "${repo_root}" -B "${build_dir}" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_TESTING=OFF \
+  -DTVM_BUILD_PYTHON_MODULE=ON \
+  -DUSE_CUDA=/usr/local/cuda \
+  -DUSE_LLVM=OFF \
+  -DUSE_CUBLAS=OFF -DUSE_CUDNN=OFF -DUSE_CUTLASS=OFF -DUSE_NCCL=OFF -DUSE_NVTX=OFF \
+  -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+cmake --build "${build_dir}" --target tvm_runtime tvm_runtime_cuda --parallel "${parallel}"
+
+cuda_lib="${build_dir}/lib/libtvm_runtime_cuda.so"
+test -f "${cuda_lib}"
+patchelf --set-rpath '$ORIGIN' "${cuda_lib}"
+echo "CUDA runtime: ${cuda_lib}"

ci/scripts/package/windows_build_libtvm_runtime_cuda.bat

@@ -0,0 +1,99 @@
+@echo off
+rem Licensed to the Apache Software Foundation (ASF) under one
+rem or more contributor license agreements.  See the NOTICE file
+rem distributed with this work for additional information
+rem regarding copyright ownership.  The ASF licenses this file
+rem to you under the Apache License, Version 2.0 (the
+rem "License"); you may not use this file except in compliance
+rem with the License.  You may obtain a copy of the License at
+rem
+rem   http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing,
+rem software distributed under the License is distributed on an
+rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+rem KIND, either express or implied.  See the License for the
+rem specific language governing permissions and limitations
+rem under the License.
+rem
+rem Build tvm_runtime_cuda.dll on a Windows runner, run by the build_cuda_runtime
+rem CI job (on the host; unlike Linux there is no build container on Windows).
+rem Installs the pinned CUDA toolkit via conda and builds the sidecar into
+rem build-wheel-cuda\lib\ for the wheel build to bundle. Windows mirror of
+rem manylinux_build_libtvm_runtime_cuda.sh. Run with: shell: cmd.
+setlocal enableextensions
+
+rem repo root = this script's directory / ..\..\.. (native Windows paths; no cygpath).
+pushd "%~dp0..\..\.." || exit /b 1
+set "repo_root=%CD%"
+popd
+set "build_dir=%repo_root%\build-wheel-cuda"
+set "cuda_prefix=C:\opt\cuda"
+
+rem Locate conda: the runner ships Miniconda (exposed via %CONDA%) but it may not be
+rem on PATH in this shell.
+set "conda_exe=conda"
+where conda >nul 2>nul || set "conda_exe=%CONDA%\Scripts\conda.exe"
+
+rem Install the pinned CUDA toolkit via conda from the nvidia channel, mirroring the
+rem LLVM-via-conda install used elsewhere. The win-64 channel caps at 13.0.x, so this
+rem pins 13.0.2 -- slightly behind the Linux image's CUDA 13.1, which is harmless: the
+rem sidecar has no device code and links the CUDA runtime by soname only. The nvidia
+rem CDN occasionally returns a transient HTTP 5xx, so retry once; a half-finished first
+rem attempt can leave the prefix partially populated, so wipe it before retrying.
+if not exist "%cuda_prefix%\Library\bin\nvcc.exe" (
+  call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y
+  if errorlevel 1 (
+    if exist "%cuda_prefix%" rmdir /s /q "%cuda_prefix%"
+    call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y || exit /b 1
+  )
+)
+
+rem conda lays the Windows toolkit out under <prefix>\Library (bin\nvcc.exe,
+rem lib\x64\cudart.lib, include\...). Discover the root from nvcc.exe so TVM's
+rem FindCUDA MSVC branch resolves against the real layout instead of a hardcode.
+set "nvcc_exe="
+for /f "delims=" %%i in ('dir /s /b "%cuda_prefix%\nvcc.exe" 2^>nul') do if not defined nvcc_exe set "nvcc_exe=%%i"
+if not defined nvcc_exe ( echo nvcc.exe not found under %cuda_prefix% & exit /b 1 )
+rem cuda_root = dirname(dirname(nvcc)) = <prefix>\Library
+for %%i in ("%nvcc_exe%") do set "nvcc_bin=%%~dpi"
+pushd "%nvcc_bin%.." || exit /b 1
+set "cuda_root=%CD%"
+popd
+set "CUDA_PATH=%cuda_root%"
+
+python -m pip install -U pip cmake ninja || exit /b 1
+"%nvcc_exe%" --version || exit /b 1
+
+rem nvcc needs the MSVC host compiler (cl.exe), so locate VS via vswhere and run the
+rem cmake configure+build inside vcvars64 (this shell is not a VS Developer prompt).
+set "vswhere=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"
+set "vs_path="
+for /f "usebackq delims=" %%i in (`"%vswhere%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "vs_path=%%i"
+if not defined vs_path ( echo Visual Studio with VC tools not found & exit /b 1 )
+set "vcvars=%vs_path%\VC\Auxiliary\Build\vcvars64.bat"
+
+if exist "%build_dir%" rmdir /s /q "%build_dir%"
+
+rem CMAKE_CUDA_COMPILER only tells CMake which nvcc to use (load-bearing: the conda
+rem nvcc is not on PATH); it does not affect the resulting tvm_runtime_cuda.dll, which
+rem is built only from .cc host sources (no .cu device code). CMAKE_CUDA_ARCHITECTURES
+rem is intentionally not set -- a no-op for the same reason, and modern CMake fills a
+rem default. -allow-unsupported-compiler guards against the runner's MSVC being newer
+rem than the CUDA toolkit officially supports. The cmake command is kept on one line:
+rem `^` continuations in a batch file break on any trailing whitespace.
+rem CMake parses backslashes in string values as escapes (e.g. C:\opt -> invalid \o),
+rem so hand cmake forward-slash paths. cmd builtins (rmdir / if exist) keep backslashes.
+set "repo_root_fwd=%repo_root:\=/%"
+set "build_dir_fwd=%build_dir:\=/%"
+set "cuda_root_fwd=%cuda_root:\=/%"
+set "nvcc_fwd=%nvcc_exe:\=/%"
+
+call "%vcvars%" || exit /b 1
+cmake -S "%repo_root_fwd%" -B "%build_dir_fwd%" -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DTVM_BUILD_PYTHON_MODULE=ON -DUSE_CUDA="%cuda_root_fwd%" -DUSE_LLVM=OFF -DUSE_CUBLAS=OFF -DUSE_CUDNN=OFF -DUSE_CUTLASS=OFF -DUSE_NCCL=OFF -DUSE_NVTX=OFF -DCMAKE_CUDA_COMPILER="%nvcc_fwd%" -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler" || exit /b 1
+cmake --build "%build_dir_fwd%" --target tvm_runtime tvm_runtime_cuda --config Release || exit /b 1
+
+if not exist "%build_dir%\lib\tvm_runtime_cuda.dll" ( echo tvm_runtime_cuda.dll was not produced & exit /b 1 )
+rem No patchelf/rpath step on Windows; delvewheel vendors dependencies at repair time.
+echo CUDA runtime: %build_dir%\lib\tvm_runtime_cuda.dll
+endlocal