""" Script to update a Github release's assets. Given a Github release ID (e.g. `prerelease` or `0.9.0`), this script will fetch the associated binary assets from our cloud storage (`build.rerun.io`) and upload them to the release as native assets. This is expected to be run by the release & pre-release workflows. You can also run it manually if you want to update a specific release's assets: python scripts/ci/sync_release_assets.py --github-release prerelease --github-token --update """ from __future__ import annotations import argparse import sys import time from typing import TYPE_CHECKING, cast from github import Github from google.cloud import storage if TYPE_CHECKING: from github.GitRelease import GitRelease from github.Repository import Repository Assets = dict[str, storage.Blob] MissingAssets = list[tuple[str, str]] def get_any_release(repo: Repository, tag_name: str) -> GitRelease | None: """Fetch any release from a Github repository, even drafts.""" # Do _not_ use `repo.get_release`, it silently ignores drafts. return next(rel for rel in repo.get_releases() if rel.tag_name == tag_name) def fetch_binary_assets( tag: str, commit: str, *, do_wheels: bool = True, do_rerun_c: bool = True, do_rerun_cpp_sdk: bool = True, do_rerun_cli: bool = True, do_rerun_js: bool = True, ) -> Assets: """Given a release ID, fetches all associated binary assets from our cloud storage (build.rerun.io).""" assets = {} gcs = storage.Client() bucket = gcs.bucket("rerun-builds") commit_short = commit[:7] print(f"Fetching the following binary assets for #{commit_short}:") if do_wheels: print(" - Python wheels") if do_rerun_c: print(" - C libs") if do_rerun_cpp_sdk: print(" - C++ uber SDK") if do_rerun_cli: print(" - CLI (Viewer)") if do_rerun_js: print(" - JS package") missing_assets: MissingAssets = [] def report_missing_blob(asset_name: str, blob_url: str) -> None: print(f"Missing {asset_name}: gs://{bucket.name}/{blob_url}") missing_assets.append((asset_name, blob_url)) # Python wheels if do_wheels: found = False wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) for blob in (bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")): if blob is not None and blob.name is not None: name = blob.name.split("/")[-1] # NOTE(cmc): I would love to rename those so they match the versioning of our # other assets, but that breaks `pip install`… # if "macosx" in name: # if "x86_64" in name: # name = f"rerun_sdk-{tag}-aarch64-apple-darwin.whl" # if "arm64" in name: # name = f"rerun_sdk-{tag}-x86_64-apple-darwin.whl" # # if "manylinux_2_28_x86_64" in name: # if "x86_64" in name: # name = f"rerun_sdk-{tag}-x86_64-unknown-linux-gnu.whl" # # if "win_amd64" in name: # name = f"rerun_sdk-{tag}-x86_64-pc-windows-msvc.whl" print(f"Found Python wheels: {name}") found = True assets[name] = blob if not found: report_missing_blob("Python wheels", f"commit/{commit_short}/wheels/*.whl") # rerun_c if do_rerun_c: rerun_c_blobs = [ ( f"rerun_c-{tag}-x86_64-pc-windows-msvc.lib", f"commit/{commit_short}/rerun_c/windows-x64/rerun_c.lib", ), ( f"librerun_c-{tag}-x86_64-unknown-linux-gnu.a", f"commit/{commit_short}/rerun_c/linux-x64/librerun_c.a", ), ( f"librerun_c-{tag}-aarch64-unknown-linux-gnu.a", f"commit/{commit_short}/rerun_c/linux-arm64/librerun_c.a", ), ( f"librerun_c-{tag}-aarch64-apple-darwin.a", f"commit/{commit_short}/rerun_c/macos-arm64/librerun_c.a", ), ] for name, blob_url in rerun_c_blobs: blob = bucket.get_blob(blob_url) if blob is not None: print(f"Found Rerun C library: {name}") assets[name] = blob else: report_missing_blob(name, blob_url) # rerun_cpp_sdk if do_rerun_cpp_sdk: rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip") for blob in [rerun_cpp_sdk_blob]: if blob is not None and blob.name is not None: name = blob.name.split("/")[-1] print(f"Found Rerun cross-platform bundle: {name}") assets[f"rerun_cpp_sdk-{tag}-multiplatform.zip"] = blob # Upload again as rerun_cpp_sdk.zip for convenience. # # ATTENTION: Renaming this file has tremendous ripple effects: # Not only is this the convenient short name we use in examples, # we also rely on https://github.com/rerun-io/rerun/releases/latest/download/rerun_cpp_sdk.zip # to always give you the latest stable version of the Rerun SDK. # -> The name should *not* contain the version number. assets["rerun_cpp_sdk.zip"] = blob else: report_missing_blob( f"rerun_cpp_sdk-{tag}-multiplatform.zip", f"commit/{commit_short}/rerun_cpp_sdk.zip", ) # rerun-cli if do_rerun_cli: rerun_cli_blobs = [ ( f"rerun-cli-{tag}-x86_64-pc-windows-msvc.exe", f"commit/{commit_short}/rerun-cli/windows-x64/rerun.exe", ), ( f"rerun-cli-{tag}-x86_64-unknown-linux-gnu", f"commit/{commit_short}/rerun-cli/linux-x64/rerun", ), ( f"rerun-cli-{tag}-aarch64-unknown-linux-gnu", f"commit/{commit_short}/rerun-cli/linux-arm64/rerun", ), ( f"rerun-cli-{tag}-aarch64-apple-darwin", f"commit/{commit_short}/rerun-cli/macos-arm64/rerun", ), ( f"Rerun-{tag}-aarch64-apple-darwin.app.tar.gz", f"commit/{commit_short}/rerun-cli/macos-arm64/rerun", ), ] for name, blob_url in rerun_cli_blobs: blob = bucket.get_blob(blob_url) if blob is not None: print(f"Found Rerun CLI binary: {name}") assets[name] = blob else: report_missing_blob(name, blob_url) # rerun-js if do_rerun_js: # note: we don't include the version tag in the asset name here, # otherwise `latest` downloads contain the version number. rerun_js_blobs = [ ( "rerun-js-web-viewer.tar.gz", f"commit/{commit_short}/rerun_js/web-viewer.tar.gz", ), ( "rerun-js-web-viewer-react.tar.gz", f"commit/{commit_short}/rerun_js/web-viewer-react.tar.gz", ), ] for name, blob_url in rerun_js_blobs: blob = bucket.get_blob(blob_url) if blob is not None: print(f"Found Rerun JS package: {name}") assets[name] = blob else: report_missing_blob(name, blob_url) if missing_assets: missing_blob_paths = "\n".join( f" - {asset_name}: gs://{bucket.name}/{blob_url}" for asset_name, blob_url in missing_assets ) raise RuntimeError( f"Failed to fetch {len(missing_assets)} requested release asset(s) from GCS for " f"tag {tag!r} at commit {commit_short}.\n" f"Missing blobs:\n{missing_blob_paths}\n" "Check that the release build workflow completed successfully and uploaded these artifacts." ) return assets def remove_release_assets(release: GitRelease) -> None: print("Removing pre-existing release assets…") for asset in release.get_assets(): print(f"Removing {asset.name}…") asset.delete_asset() def update_release_assets(release: GitRelease, assets: Assets) -> None: print("Updating release assets…") for name, blob in assets.items(): blob_contents = blob.download_as_bytes() # NOTE: Do _not_ ever use `blob.size`, it might or might not give you the size you expect # depending on the versions of your gcloud dependencies, which in turn might or might not fail # the upload in all kinds of unexpected ways (including SSL errors!) depending on the versions # of your reqwest & pygithub dependencies. blob_raw_size = len(blob_contents) print(f"Uploading {name} ({blob_raw_size} bytes)…") release.upload_asset_from_memory( blob_contents, blob_raw_size, name, content_type="application/octet-stream", ) def main() -> None: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--github-token", required=True, help="GitHub token") parser.add_argument("--github-repository", default="rerun-io/rerun", help="GitHub repository") parser.add_argument( "--github-release", required=True, help="Github (pre)release tag (e.g. `prerelease` or `0.9.0`)", ) parser.add_argument("--github-timeout", default=120, help="Timeout for Github related operations") parser.add_argument("--wait", default=0, help="Sleep a bit before doing anything") parser.add_argument( "--remove", action="store_true", help="Remove existing assets from the specified release", ) parser.add_argument( "--update", action="store_true", help="Update new assets to the specified release", ) parser.add_argument("--no-wheels", action="store_true", help="Don't upload Python wheels") parser.add_argument("--no-rerun-c", action="store_true", help="Don't upload C libraries") parser.add_argument("--no-rerun-cpp-sdk", action="store_true", help="Don't upload C++ uber SDK") parser.add_argument("--no-rerun-cli", action="store_true", help="Don't upload CLI") parser.add_argument("--no-rerun-js", action="store_true", help="Don't upload JS package") args = parser.parse_args() # Wait for a bit before doing anything, if you must. wait_time_secs = float(args.wait) if wait_time_secs > 0.0: print(f"Waiting for {wait_time_secs}s…") time.sleep(wait_time_secs) gh = Github(args.github_token, timeout=args.github_timeout) repo = gh.get_repo(args.github_repository) release = cast("GitRelease", get_any_release(repo, args.github_release)) commit = {tag.name: tag.commit for tag in repo.get_tags()}[args.github_release] if release.body is None: print( "Release has no body - make sure to add release notes! " "You might also run into this if you created the release manually " "and not from the draft created by the release job, please check.", file=sys.stderr, ) sys.exit(1) print( f'Syncing binary assets for release `{release.tag_name}` ("{release.title}" @{release.published_at} draft={release.draft}) #{commit.sha[:7]}…', ) assets = fetch_binary_assets( release.tag_name, commit.sha, do_wheels=not args.no_wheels, do_rerun_c=not args.no_rerun_c, do_rerun_cpp_sdk=not args.no_rerun_cpp_sdk, do_rerun_cli=not args.no_rerun_cli, do_rerun_js=not args.no_rerun_js, ) if args.remove: remove_release_assets(release) if args.update: update_release_assets(release, assets) # Github will unconditionally draft a release in some cases (e.g. because the branch it has # originated from has been modified since). This is beyond our control. # # Draft releases are not accessible through any of the expected ways, so we make sure to fix # that. # # See e.g. . if release.draft: print("Detected mistakenly drafted release, undrafting…") release.update_release(release.title, release.body, draft=False, prerelease=release.prerelease) if __name__ == "__main__": main()