chore: import upstream snapshot with attribution

scripts/download_transformer_model.py

@@ -0,0 +1,7 @@
+from huggingface_hub import snapshot_download
+import sys
+
+print(f"Downloading model files for {sys.argv[1]}...")
+snapshot_download(
+    repo_id=sys.argv[1], allow_patterns=["*.txt", "*.json", "*.h5", "*.model"]
+)

scripts/evaluate_release_tag.py

@@ -0,0 +1,74 @@
+"""Evaluate release tag for whether docs should be built or not.
+
+"""
+import argparse
+from subprocess import check_output
+from typing import List
+
+from pep440_version_utils import Version, is_valid_version
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Parse all the command line arguments for the release script."""
+
+    parser = argparse.ArgumentParser(description="Evaluate whether docs should be built for release tag.")
+    parser.add_argument(
+        "tag",
+        type=str,
+        help="The tag currently being released.",
+    )
+
+    return parser
+
+
+def is_plain_version(version: Version) -> bool:
+    """Check whether a version is a X.x.x release with no alpha or rc markers"""
+    return version.base_version == str(version)
+
+
+def git_existing_tag_versions() -> List[Version]:
+    """Return all existing tags in the local git repo."""
+    stdout = check_output(["git", "tag"])
+    tags = set(stdout.decode().split("\n"))
+    versions = [Version(tag) for tag in tags if is_valid_version(tag)]
+    return versions
+
+
+def git_plain_tag_versions(versions: List[Version]) -> List[Version]:
+    """Return non-alpha/rc existing tags"""
+    return [version for version in versions if is_plain_version(version)]
+
+def filter_ga_relases(tags: List[Version]) -> List[Version]:
+    """Return all general availability (GA) releases"""
+    return [tag for tag in tags if tag.is_alpha is False and tag.is_beta is False and tag.is_release_candidate is False]
+
+
+def should_build_docs(tag: Version) -> bool:
+    """Docs should be built only for the latest GA release tags"""
+    existing_tags = git_existing_tag_versions()
+
+    ga_releases = filter_ga_relases(existing_tags)
+    ga_releases.sort()
+    latest_version = ga_releases[-1]
+    print(f"The latest General Availability release tag is {latest_version}.")
+    need_to_build_docs = False
+
+    if not is_plain_version(tag):
+        print(f"Tag {tag} is an alpha, beta, rc, nightly, or otherwise non-standard version.")
+    elif tag >= latest_version:
+        print(f"Tag {tag} is the latest version. Docs should be built.")
+        need_to_build_docs = True
+
+    return need_to_build_docs
+
+
+def main(args: argparse.Namespace) -> None:
+    tag = Version(args.tag)
+    if not should_build_docs(tag):
+        raise SystemExit(f"Docs should not be built for tag {tag}.")
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    main(cmdline_args)

scripts/lint_changelog_files.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+# Lint changelog filenames to avoid merging of incorrectly named changelog fragment files
+# For more info about proper changelog file naming, see https://github.com/RasaHQ/rasa/blob/main/changelog/README.md
+FILES=`ls -A changelog/ | grep -v -E "^([0-9]+.(misc|doc|improvement|feature|removal|bugfix).md|README.md|_template.md.jinja2)$" || true`
+if [ -z "$FILES" ]
+then
+    echo "Changelog files are properly configured."
+else
+    echo "Unexpected files under the changelog/ folder:"
+    echo $FILES
+    exit 1
+fi

scripts/lint_python_docstrings.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -euo pipefail
+# Lint docstrings only against the the diff to avoid too many errors.
+# Check only production code. Ignore other other errors which are captured by `lint`
+
+# Compare against `main` if no branch was provided
+BRANCH="${1:-main}"
+# Diff of committed changes (shows only changes introduced by your branch
+FILES_WITH_DIFF=`git diff $BRANCH...HEAD --name-only -- rasa/**/*.py | xargs echo -n`
+
+if [ ! -z "$FILES_WITH_DIFF" ]
+then
+    poetry run ruff check --select D --diff $FILES_WITH_DIFF
+else
+    echo "No python files in diff."
+fi
+
+# Diff of uncommitted changes for running locally
+DEV_FILES_WITH_DIFF=`git diff HEAD --name-only -- rasa | xargs echo -n`
+
+if [ ! -z "$DEV_FILES_WITH_DIFF" ]
+then
+    poetry run ruff check --select D --diff $DEV_FILES_WITH_DIFF
+fi

scripts/ping_slack_about_package_release.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+if [[ ${GITHUB_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    curl -X POST -H "Content-type: application/json" \
+	 --data "{\"text\":\"💥 New *Rasa Open Source* version ${GITHUB_TAG} has been released! https://github.com/RasaHQ/rasa/releases/tag/${GITHUB_TAG}\"}" \
+	 "https://hooks.slack.com/services/T0GHWFTS8/BMTQQL47K/${SLACK_WEBHOOK_TOKEN}"
+fi

scripts/poetry-version.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""Extract the poetry version from .github/poetry_version.txt. Used in e.g. github workflows."""
+
+import pathlib
+import sys
+import re
+
+# The poetry version is stored in the .github/poetry_version.txt file due to the https://github.com/python-poetry/poetry/issues/3316
+poetry_version_txt = pathlib.Path(__file__).parent.parent / ".github" / "poetry_version.txt"
+
+if __name__ == "__main__":
+    version_regex = r'poetry-version=(.*)'
+    with poetry_version_txt.open() as f:
+        for line in f:
+            m = re.search(version_regex, line)
+            if m:
+                print(m.group(1))
+                sys.exit(0)
+        else:
+            print("Failed to find poetry version.")
+            sys.exit(1)

scripts/prepare_nightly_release.py

@@ -0,0 +1,103 @@
+"""Prepare a Rasa nightly release.
+
+- increases the version number to dev version provided
+"""
+import argparse
+import os
+import sys
+import toml
+from pathlib import Path
+from typing import Text
+from pep440_version_utils import Version, is_valid_version
+
+VERSION_FILE_PATH = "rasa/version.py"
+
+PYPROJECT_FILE_PATH = "pyproject.toml"
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Parse all the command line arguments for the release script."""
+
+    parser = argparse.ArgumentParser(description="prepare the next nightly release")
+    parser.add_argument(
+        "--next_version", type=str, help="Rasa nightly version number",
+    )
+
+    return parser
+
+
+def project_root() -> Path:
+    """Root directory of the project."""
+    return Path(os.path.dirname(__file__)).parent
+
+
+def write_version_file(path: Text, version: Version) -> None:
+    """Dump a new version into the python version file."""
+
+    version_file = project_root() / path
+
+    with version_file.open("w") as f:
+        f.write(
+            f"# this file will automatically be changed,\n"
+            f"# do not add anything but the version number here!\n"
+            f'__version__ = "{version}"\n'
+        )
+
+
+def write_version_to_pyproject(pyproject_file_path: Text, version: Version) -> None:
+    """Dump a new version into the pyproject.toml."""
+
+    pyproject_file = project_root() / pyproject_file_path
+
+    try:
+        data = toml.load(pyproject_file)
+        data["tool"]["poetry"]["name"] = "rasa-nightly"
+        data["tool"]["poetry"]["version"] = str(version)
+        data["tool"]["poetry"]["packages"] = [{"include" : "rasa"},]
+        with pyproject_file.open("w", encoding="utf8") as f:
+            toml.dump(data, f)
+    except (FileNotFoundError, TypeError):
+        print(f"Unable to update {pyproject_file}: file not found.")
+        sys.exit(1)
+    except toml.TomlDecodeError:
+        print(f"Unable to parse {pyproject_file}: incorrect TOML file.")
+        sys.exit(1)
+
+
+def parse_next_version(version: Text) -> Version:
+    """Find the next version as a proper semantic version string."""
+    return Version(version)
+
+
+def next_version(args: argparse.Namespace) -> Version:
+    """Take cmdline args or ask the user for the next version and return semver."""
+    return parse_next_version(args.next_version)
+
+
+def print_done_message(version: Version) -> None:
+    """Print final information for the user on what to do next."""
+
+    print()
+    print(f"\033[94m All done - changes for rasa nightly version {version} are ready! \033[0m")
+    print()
+
+
+def main(args: argparse.Namespace) -> None:
+    """Start a nightly release preparation."""
+
+    print(
+        "The release script will temporarily change the version number to a nightly version. Let's go!"
+    )
+
+    version = next_version(args)
+
+    write_version_file(VERSION_FILE_PATH, version)
+    write_version_to_pyproject(PYPROJECT_FILE_PATH, version)
+
+    print_done_message(version)
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    main(cmdline_args)

scripts/publish_gh_release_notes.py

@@ -0,0 +1,106 @@
+"""
+Script used to publish GitHub release notes extracted from CHANGELOG.mdx.
+This script is executed by GitHub after a new release was successfully built.
+
+Uses the following environment variables:
+* GITHUB_TAG: the name of the tag of the current commit.
+* GITHUB_TOKEN: a personal access token with 'repo' permissions.
+
+The script also requires ``pandoc`` to be previously installed in the system.
+Requires Python3.6+.
+
+Based on code from pytest.
+https://github.com/pytest-dev/pytest/blob/master/scripts/publish_gh_release_notes.py
+Copyright Holger Krekel and others, 2004-2019.
+
+Distributed under the terms of the MIT license, pytest is free and open source software.
+"""
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Text
+
+# if this needs any more dependencies, they need to be installed on github deploy stage
+import github3
+from pep440_version_utils import Version
+
+
+def create_github_release(slug: Text, token: Text, tag_name: Text, body: Text):
+    """Create a github release."""
+
+    github = github3.login(token=token)
+    owner, repo = slug.split("/")
+    repo = github.repository(owner, repo)
+    return repo.create_release(tag_name=tag_name, body=body)
+
+
+def parse_changelog(tag_name: Text) -> Text:
+    """Read the changelog and extract the most recently release entry."""
+
+    p = Path(__file__).parent.parent / "CHANGELOG.mdx"
+    changelog_lines = p.read_text(encoding="UTF-8").splitlines()
+
+    title_regex = re.compile(r"##\s*\[(\d+\.\d+\.\d+)(\S*)\]\s*-\s*\d{4}-\d{2}-\d{2}")
+    consuming_version = False
+    version_lines = []
+    for line in changelog_lines:
+        m = title_regex.match(line)
+        if m:
+            # found the version we want: start to consume lines
+            # until we find the next version title
+            if m.group(1) == tag_name:
+                consuming_version = True
+            # found a new version title while parsing the version we want: break out
+            elif consuming_version:
+                break
+        if consuming_version:
+            version_lines.append(line)
+
+    if consuming_version:
+        # drop the first lines (version headline, not needed for GH)
+        return "\n".join(version_lines[2:]).strip()
+    else:
+        return None
+
+
+def main():
+    tag_name = os.environ.get("GITHUB_TAG")
+    if not tag_name:
+        print("environment variable GITHUB_TAG not set", file=sys.stderr)
+        return 1
+
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("GITHUB_TOKEN not set", file=sys.stderr)
+        return 1
+
+    slug = os.environ.get("GITHUB_REPO_SLUG")
+    if not slug:
+        print("GITHUB_REPO_SLUG not set", file=sys.stderr)
+        return 1
+
+    version = Version(tag_name)
+    if version.pre:
+        md_body = "_Pre-release version_"
+    else:
+        md_body = parse_changelog(tag_name)
+
+    if md_body is None:
+        print("Failed to extract changelog entries for version from changelog.")
+        return 2
+
+    if not create_github_release(slug, token, tag_name, md_body):
+        print("Could not publish release notes:", file=sys.stderr)
+        print(md_body, file=sys.stderr)
+        return 5
+
+    print()
+    print(f"Release notes for {tag_name} published successfully:")
+    print(f"https://github.com/{slug}/releases/tag/{tag_name}")
+    print()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/push_docs_to_branch.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# In case this script fails in our CI workflows, you can run it locally.
+# For instance, if the doc version for 2.2.10 fails to be pushed, you
+# can do:
+#
+# 1. make install install-docs
+# 2. make prepare-docs
+# 3. TMP_DOCS_FOLDER=/tmp/documentation DOCS_BRANCH=documentation GITHUB_REF=refs/tags/2.2.10 GITHUB_REPOSITORY=RasaHQ/rasa ./scripts/push_docs_to_branch.sh
+#
+# This script will push the changes to the `documentation` branch of this repository.
+
+set -Eeuo pipefail
+
+TODAY=`date "+%Y%m%d"`
+# we build new versions only for minors and majors
+PATTERN_FOR_NEW_VERSION="^refs/tags/[0-9]+\\.0\\.0$"
+PATTERN_FOR_EXISTING_VERSION="^refs/tags/[0-9]+\\.[0-9]+\\.[0-9]+$"
+MAIN_REF=refs/heads/main
+VARIABLES_JSON=docs/docs/variables.json
+SOURCES_FILES=docs/docs/sources/
+REFERENCE_FILES=docs/docs/reference/
+CHANGELOG=docs/docs/changelog.mdx
+TELEMETRY_REFERENCE=docs/docs/telemetry/reference.mdx
+
+[[ ! $GITHUB_REF =~ $PATTERN_FOR_NEW_VERSION ]] \
+&& [[ ! $GITHUB_REF =~ $PATTERN_FOR_EXISTING_VERSION ]] \
+&& [[ $GITHUB_REF != $MAIN_REF ]] \
+&& echo "Not on main or tagged version, skipping." \
+&& exit 0
+
+NEW_VERSION=
+EXISTING_VERSION=
+if [[ "$GITHUB_REF" =~ $PATTERN_FOR_NEW_VERSION ]]
+then
+    NEW_VERSION=$(echo $GITHUB_REF | sed -E "s/^refs\/tags\/([0-9]+)\.([0-9]+)\.0$/\1.x/")
+    if [[ -n ${CI} ]]; then echo "New version: ${NEW_VERSION}"; fi
+elif [[ "$GITHUB_REF" =~ $PATTERN_FOR_EXISTING_VERSION ]]
+then
+    EXISTING_VERSION=$(echo $GITHUB_REF | sed -E "s/^refs\/tags\/([0-9]+)\.([0-9]+)\.[0-9]+$/\1.x/")
+    if [[ -n ${CI} ]]; then echo "Existing version: ${EXISTING_VERSION}"; fi
+fi
+
+# clone the $DOCS_BRANCH in a temp directory
+git clone --depth=1 --branch=$DOCS_BRANCH git@github.com:$GITHUB_REPOSITORY.git $TMP_DOCS_FOLDER
+
+if [ ! -z "$NEW_VERSION" ] && [ -d "$TMP_DOCS_FOLDER/docs/versioned_docs/version-$NEW_VERSION/" ]
+then
+    echo "Trying to create a new docs version, but the folder already exist. Updating the $NEW_VERSION version instead..."
+    EXISTING_VERSION=$NEW_VERSION
+fi
+
+# install yarn dependencies in the temp directory
+cd $TMP_DOCS_FOLDER/docs && yarn install && cd - || exit 1
+
+if [ ! -z "$EXISTING_VERSION" ]
+then
+    echo "Updating docs for existing version $EXISTING_VERSION..."
+    # FIXME: this doesn't support all types of docs updates on an existing version at the moment,
+    # For instance if we were to make significant updates to the documentation pages
+    # (creating new page, deleting some, updating the sidebar), these changes wouldn't work here.
+    cp -R docs/docs/* $TMP_DOCS_FOLDER/docs/versioned_docs/version-$EXISTING_VERSION/
+else
+    echo "Updating the docs..."
+    # remove everything in the previous docs/ folder, except versioned_docs/*, versioned_sidebars/*
+    # and versions.js files.
+    # skip node_modules/ because `yarn install` has run
+    cd $TMP_DOCS_FOLDER/docs/ && find  . ! -path . ! -regex '.*\(version\|node_modules\).*' -exec rm -rf {} + && cd - || exit 1
+    # update the docs/ folder with the latest version of the docs
+    cp -R `ls -A | grep -v "^\.git$"` $TMP_DOCS_FOLDER/
+
+    if [ ! -z "$NEW_VERSION" ]
+    then
+        echo "Generating docs for new version $NEW_VERSION..."
+        cd $TMP_DOCS_FOLDER/docs && yarn run new-version $NEW_VERSION && cd - || exit 1
+    fi
+fi
+
+CURRENTLY_EDITING_VERSION=${EXISTING_VERSION:-$NEW_VERSION}
+if [ -n "$CURRENTLY_EDITING_VERSION" ]
+then
+    cd $TMP_DOCS_FOLDER/docs && yarn run update-versioned-sources $CURRENTLY_EDITING_VERSION && cd - || exit 1
+fi
+
+cd $TMP_DOCS_FOLDER
+
+if [ -z "$(git status --porcelain)" ]
+then
+    echo "Nothing changed in docs, done 👍"
+else
+    echo "Pushing changes to git..."
+    git add .
+    git add --force $VARIABLES_JSON $SOURCES_FILES $CHANGELOG $REFERENCE_FILES $TELEMETRY_REFERENCE
+    git commit -am "AUTO docusaurus $TODAY"
+    git fetch --unshallow
+    git push origin $DOCS_BRANCH
+
+    echo "Done 👌"
+fi

scripts/read_tensorflow_version.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""Extract the TensorFlow version from poetry.lock. Used in e.g. github workflows."""
+
+import pathlib
+import sys
+import toml
+
+poetry_path = 'poetry.lock'
+
+if __name__ == "__main__":
+    if pathlib.Path(poetry_path).is_file():
+      # Get the list of dict that contains all information about packages
+      package_list = toml.load(poetry_path).get('package')
+      # Extract tensorflow version
+      tf_version = next(filter(lambda x: x.get('name') == 'tensorflow', package_list)).get('version')
+      print(tf_version)
+      sys.exit(0)
+
+    else:
+      print(f'File {poetry_path} does not exist.')
+      sys.exit(1)

scripts/release.py

@@ -0,0 +1,364 @@
+"""Prepare a Rasa OSS release.
+
+- creates a release branch
+- creates a new changelog section in CHANGELOG.mdx based on all collected changes
+- increases the version number
+- pushes the new branch to GitHub
+"""
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+from subprocess import CalledProcessError, check_call, check_output
+from typing import Text, Set
+
+import questionary
+import toml
+from pep440_version_utils import Version, is_valid_version
+
+
+VERSION_FILE_PATH = "rasa/version.py"
+
+PYPROJECT_FILE_PATH = "pyproject.toml"
+
+REPO_BASE_URL = "https://github.com/RasaHQ/rasa"
+
+RELEASE_BRANCH_PREFIX = "prepare-release-"
+
+PRERELEASE_FLAVORS = ("alpha", "rc")
+
+RELEASE_BRANCH_PATTERN = re.compile(r"^\d+\.\d+\.x$")
+
+PUBLIC_REMOTE = "public"
+DEFAULT_REMOTE = "origin"
+FIRST_CALM_VERSION = "3.7.0"
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Parse all the command line arguments for the release script."""
+
+    parser = argparse.ArgumentParser(description="prepare the next library release")
+    parser.add_argument(
+        "--next_version",
+        type=str,
+        help="Either next version number or 'major', 'minor', 'micro', 'alpha', 'rc'",
+    )
+
+    return parser
+
+
+def project_root() -> Path:
+    """Root directory of the project."""
+    return Path(os.path.dirname(__file__)).parent
+
+
+def version_file_path() -> Path:
+    """Path to the python file containing the version number."""
+    return project_root() / VERSION_FILE_PATH
+
+
+def pyproject_file_path() -> Path:
+    """Path to the pyproject.toml."""
+    return project_root() / PYPROJECT_FILE_PATH
+
+
+def write_version_file(version: Version) -> None:
+    """Dump a new version into the python version file."""
+
+    with version_file_path().open("w") as f:
+        f.write(
+            f"# this file will automatically be changed,\n"
+            f"# do not add anything but the version number here!\n"
+            f'__version__ = "{version}"\n'
+        )
+    check_call(["git", "add", str(version_file_path().absolute())])
+
+
+def write_version_to_pyproject(version: Version) -> None:
+    """Dump a new version into the pyproject.toml."""
+    pyproject_file = pyproject_file_path()
+
+    try:
+        data = toml.load(pyproject_file)
+        data["tool"]["poetry"]["version"] = str(version)
+        with pyproject_file.open("w", encoding="utf8") as f:
+            toml.dump(data, f)
+    except (FileNotFoundError, TypeError):
+        print(f"Unable to update {pyproject_file}: file not found.")
+        sys.exit(1)
+    except toml.TomlDecodeError:
+        print(f"Unable to parse {pyproject_file}: incorrect TOML file.")
+        sys.exit(1)
+
+    check_call(["git", "add", str(pyproject_file.absolute())])
+
+
+def get_current_version() -> Text:
+    """Return the current library version."""
+
+    if not version_file_path().is_file():
+        raise FileNotFoundError(
+            f"Failed to find version file at {version_file_path().absolute()}"
+        )
+
+    # context in which we evaluate the version py -
+    # to be able to access the defined version, it already needs to live in the
+    # context passed to exec
+    _globals = {"__version__": ""}
+    with version_file_path().open() as f:
+        exec(f.read(), _globals)
+
+    return _globals["__version__"]
+
+
+def confirm_version(version: Version) -> bool:
+    """Allow the user to confirm the version number."""
+
+    if str(version) in git_existing_tags():
+        confirmed = questionary.confirm(
+            f"Tag with version '{version}' already exists, overwrite?", default=False
+        ).ask()
+    else:
+        confirmed = questionary.confirm(
+            f"Current version is '{get_current_version()}. "
+            f"Is the next version '{version}' correct ?",
+            default=True,
+        ).ask()
+    if confirmed:
+        return True
+    else:
+        print("Aborting.")
+        sys.exit(1)
+
+
+def ask_version() -> Text:
+    """Allow the user to confirm the version number."""
+
+    def is_valid_version_number(v: Text) -> bool:
+        return v in {"major", "minor", "micro", "alpha", "rc"} or is_valid_version(v)
+
+    current_version = Version(get_current_version())
+    next_micro_version = str(current_version.next_micro())
+    next_alpha_version = str(current_version.next_alpha())
+    version = questionary.text(
+        f"What is the version number you want to release "
+        f"('major', 'minor', 'micro', 'alpha', 'rc' or valid version number "
+        f"e.g. '{next_micro_version}' or '{next_alpha_version}')?",
+        validate=is_valid_version_number,
+    ).ask()
+
+    if version in PRERELEASE_FLAVORS and not current_version.pre:
+        # at this stage it's hard to guess the kind of version bump the
+        # releaser wants, so we ask them
+        if version == "alpha":
+            choices = [
+                str(current_version.next_alpha("minor")),
+                str(current_version.next_alpha("micro")),
+                str(current_version.next_alpha("major")),
+            ]
+        else:
+            choices = [
+                str(current_version.next_release_candidate("minor")),
+                str(current_version.next_release_candidate("micro")),
+                str(current_version.next_release_candidate("major")),
+            ]
+        version = questionary.select(
+            f"Which {version} do you want to release?", choices=choices
+        ).ask()
+
+    if version:
+        return version
+    else:
+        print("Aborting.")
+        sys.exit(1)
+
+
+def get_rasa_sdk_version() -> Text:
+    """Find out what the referenced version of the Rasa SDK is."""
+
+    dependencies_filename = "pyproject.toml"
+    toml_data = toml.load(project_root() / dependencies_filename)
+    try:
+        sdk_version = toml_data["tool"]["poetry"]["dependencies"]["rasa-sdk"]
+        if isinstance(sdk_version, str):
+            return sdk_version[1:].strip()
+        else:
+            return sdk_version["version"][1:].strip()
+    except AttributeError:
+        raise Exception(f"Failed to find Rasa SDK version in {dependencies_filename}")
+
+
+def validate_code_is_release_ready(version: Version) -> None:
+    """Make sure the code base is valid (e.g. Rasa SDK is up to date)."""
+
+    sdk = Version(get_rasa_sdk_version())
+    sdk_version = (sdk.major, sdk.minor)
+    rasa_version = (version.major, version.minor)
+
+    if sdk_version != rasa_version:
+        print()
+        print(
+            f"\033[91m There is a mismatch between the Rasa SDK version ({sdk}) "
+            f"and the version you want to release ({version}). Before you can "
+            f"release Rasa OSS, you need to release the SDK and update "
+            f"the dependency. \033[0m"
+        )
+        print()
+        sys.exit(1)
+
+
+def git_existing_tags() -> Set[Text]:
+    """Return all existing tags in the local git repo."""
+
+    stdout = check_output(["git", "tag"])
+    return set(stdout.decode().split("\n"))
+
+
+def git_current_branch() -> Text:
+    """Returns the current git branch of the local repo."""
+
+    try:
+        output = check_output(["git", "symbolic-ref", "--short", "HEAD"])
+        return output.decode().strip()
+    except CalledProcessError:
+        # e.g. we are in detached head state
+        return "main"
+
+
+def git_current_branch_is_main_or_release() -> bool:
+    """
+    Returns True if the current local git
+    branch is main or a release branch e.g. 1.10.x
+    """
+    current_branch = git_current_branch()
+    return (
+        current_branch == "main"
+        or RELEASE_BRANCH_PATTERN.match(current_branch) is not None
+    )
+
+
+def create_release_branch(version: Version) -> Text:
+    """Create a new branch for this release. Returns the branch name."""
+
+    branch = f"{RELEASE_BRANCH_PREFIX}{version}"
+    check_call(["git", "checkout", "-b", branch])
+    return branch
+
+
+def create_commit(version: Version) -> None:
+    """Creates a git commit with all stashed changes."""
+    check_call(["git", "commit", "-m", f"prepared release of version {version}"])
+
+
+def push_changes(remote: str = DEFAULT_REMOTE) -> None:
+    """Pushes the current branch to the specified remote."""
+    check_call(["git", "push", remote, "HEAD"])
+
+
+def ensure_clean_git() -> None:
+    """Makes sure the current working git copy is clean."""
+
+    try:
+        check_call(["git", "diff-index", "--quiet", "HEAD", "--"])
+    except CalledProcessError:
+        print("Your git is not clean. Release script can only be run from a clean git.")
+        sys.exit(1)
+
+
+def parse_next_version(version: Text) -> Version:
+    """Find the next version as a proper semantic version string."""
+    if version == "major":
+        return Version(get_current_version()).next_major()
+    elif version == "minor":
+        return Version(get_current_version()).next_minor()
+    elif version == "micro":
+        return Version(get_current_version()).next_micro()
+    elif version == "alpha":
+        return Version(get_current_version()).next_alpha()
+    elif version == "rc":
+        return Version(get_current_version()).next_release_candidate()
+    elif is_valid_version(version):
+        return Version(version)
+    else:
+        raise Exception(f"Invalid version number '{cmdline_args.next_version}'.")
+
+
+def next_version(args: argparse.Namespace) -> Version:
+    """Take cmdline args or ask the user for the next version and return semver."""
+    return parse_next_version(args.next_version or ask_version())
+
+
+def generate_changelog(version: Version) -> None:
+    """Call tonwcrier and create a changelog from all available changelog entries."""
+    check_call(
+        ["towncrier", "build", "--yes", "--version", str(version)],
+        cwd=str(project_root()),
+    )
+
+
+def print_done_message(branch: Text, base: Text, version: Version) -> None:
+    """Print final information for the user on what to do next."""
+
+    pull_request_url = f"{REPO_BASE_URL}/compare/{base}...{branch}?expand=1"
+
+    print()
+    print(f"\033[94m All done - changes for version {version} are ready! \033[0m")
+    print()
+    print(f"Please open a PR on GitHub: {pull_request_url}")
+
+
+def print_done_message_same_branch(version: Version) -> None:
+    """
+    Print final information for the user in case changes
+    are directly committed on this branch.
+    """
+
+    print()
+    print(
+        f"\033[94m All done - changes for version {version} where committed on this branch \033[0m"
+    )
+
+
+def main(args: argparse.Namespace) -> None:
+    """Start a release preparation."""
+
+    print(
+        "The release script will increase the version number, "
+        "create a changelog and create a release branch. Let's go!"
+    )
+
+    ensure_clean_git()
+    version = next_version(args)
+    confirm_version(version)
+
+    validate_code_is_release_ready(version)
+
+    write_version_file(version)
+    write_version_to_pyproject(version)
+
+    if not version.pre:
+        # never update changelog on a prerelease version
+        generate_changelog(version)
+
+    remote = PUBLIC_REMOTE if str(version) < FIRST_CALM_VERSION else DEFAULT_REMOTE
+    # alpha workflow on feature branch when a version bump is required
+    if version.is_alpha and not git_current_branch_is_main_or_release():
+        create_commit(version)
+        push_changes(remote)
+
+        print_done_message_same_branch(version)
+    else:
+        base = git_current_branch()
+        branch = create_release_branch(version)
+
+        create_commit(version)
+        push_changes(remote)
+
+        print_done_message(branch, base, version)
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    main(cmdline_args)

scripts/write_keys_file.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+
+"""Write authentication keys from environment variable to key file for packaging."""
+
+import sys
+import re
+import os
+import json
+
+if __name__ == "__main__":
+    keys = {
+        "segment": os.environ.get("RASA_TELEMETRY_WRITE_KEY"),
+        "sentry": os.environ.get("RASA_EXCEPTION_WRITE_KEY")
+    }
+
+    with open(os.path.join("rasa", "keys"), "w") as f:
+        json.dump(keys, f)
+
+    print("Dumped keys to rasa/keys")