ci: verify assets naming

We might in the future generate automatically from the TypeScript types, generate the Python types.
For the moment, it is not typed, so we use a dict.

.dockerignore

@@ -1,9 +0,0 @@
-__pycache__/
-**/dist/*
-**/build/
-**/node_modules/
-**/tmp/*
-**/src/.venv/*
-logs/*
-*.pyc
-.DS_Store

.env.sample

@@ -8,12 +8,27 @@ LEON_LANG=en-US
 LEON_HOST=http://localhost
 LEON_PORT=1337
 
+# Enable/disable LLM
+LEON_LLM=false
+# LLM provider
+LEON_LLM_PROVIDER=local
+# LLM provider API key (if not local)
+LEON_LLM_PROVIDER_API_KEY=
+# Enable/disable LLM natural language generation
+LEON_LLM_NLG=false
+# Enable/disable LLM Action Recognition
+# Can fallback to chit-chat if no action is recognized
+LEON_LLM_ACTION_RECOGNITION=false
+
 # Time zone (current one by default)
 LEON_TIME_ZONE=
 
 # Enable/disable after speech
 LEON_AFTER_SPEECH=false
 
+# Enable/disable Leon's wake word
+LEON_WAKE_WORD=false
+
 # Enable/disable Leon's speech-to-text
 LEON_STT=false
 # Speech-to-text provider
@@ -39,7 +54,7 @@ LEON_PY_TCP_SERVER_HOST=0.0.0.0
 LEON_PY_TCP_SERVER_PORT=1342
 
 # Path to the Pipfile
-PIPENV_PIPFILE=bridges/python/src/Pipfile
+PIPENV_PIPFILE=tcp_server/src/Pipfile
 
 # Path to the virtual env in .venv/
 PIPENV_VENV_IN_PROJECT=true

.eslintrc.json

@@ -1,83 +0,0 @@
-{
-  "extends": [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/recommended",
-    "plugin:import/recommended",
-    "plugin:import/typescript",
-    "prettier"
-  ],
-  "settings": {
-    "import/resolver": {
-      "typescript": true,
-      "node": true
-    }
-  },
-  "parser": "@typescript-eslint/parser",
-  "env": {
-    "node": true,
-    "browser": true
-  },
-  "parserOptions": {
-    "ecmaVersion": 2021
-  },
-  "globals": {
-    "io": true
-  },
-  "plugins": ["@typescript-eslint", "unicorn", "import"],
-  "ignorePatterns": "*.spec.js",
-  "rules": {
-    "@typescript-eslint/no-non-null-assertion": ["off"],
-    "no-async-promise-executor": ["off"],
-    "no-underscore-dangle": ["error", { "allowAfterThis": true }],
-    "prefer-destructuring": ["error"],
-    "comma-dangle": ["error", "never"],
-    "semi": ["error", "never"],
-    "object-curly-spacing": ["error", "always"],
-    "unicorn/prefer-node-protocol": "error",
-    "@typescript-eslint/member-delimiter-style": [
-      "error",
-      {
-        "multiline": {
-          "delimiter": "none",
-          "requireLast": true
-        },
-        "singleline": {
-          "delimiter": "comma",
-          "requireLast": false
-        }
-      }
-    ],
-    "@typescript-eslint/explicit-function-return-type": "off",
-    "@typescript-eslint/consistent-type-definitions": "error",
-    "import/no-named-as-default": "off",
-    "import/no-named-as-default-member": "off",
-    "import/order": [
-      "error",
-      {
-        "groups": [
-          "builtin",
-          "external",
-          "internal",
-          "parent",
-          "sibling",
-          "index"
-        ],
-        "newlines-between": "always"
-      }
-    ]
-  },
-  "overrides": [
-    {
-      "files": ["skills/**/*.ts"],
-      "rules": {
-        "import/order": "off"
-      }
-    },
-    {
-      "files": ["*.ts"],
-      "rules": {
-        "@typescript-eslint/explicit-function-return-type": "error"
-      }
-    }
-  ]
-}

.github/CONTRIBUTING.md

@@ -77,22 +77,6 @@ npm run dev:server
 npm run dev:app
 ```
 
-### Docker
-
-```sh
-# Clone the repository
-git clone https://github.com/leon-ai/leon.git leon
-
-# Go to the project root
-cd leon
-
-# Build
-npm run docker:build
-
-# Run the development server and the development web app
-npm run docker:dev
-```
-
 ## Versioning
 
 - We use [Semantic Versioning](https://semver.org) for releases.
@@ -125,7 +109,6 @@ Scopes define high-level nodes of Leon.
 
 - bridge/python
 - bridge/nodejs
-- docker
 - hotword
 - scripts
 - server
@@ -142,6 +125,11 @@ git commit -m "chore: split training script into awesome blocks"
 git commit -m "style(web app): remove chatbot useless parentheses"
 ```
 
+### GUI Clients
+
+If you are using a GUI client such as GitKraken, you may need to disable the default Git executable to make sure to use your default shell.
+Otherwise you may encounter an error such as "npx not found".
+
 ## Sponsor
 
 You can also contribute by [sponsoring Leon](http://sponsor.getleon.ai).
@@ -190,11 +178,11 @@ Once Pyenv installed, run:
 
 ```bash
 # Install Python
-pyenv install 3.9.10 --force
-pyenv global 3.9.10
+pyenv install 3.11.9 --force
+pyenv global 3.11.9
 
 # Install Pipenv
-pip install pipenv==2022.7.24
+pip install pipenv==2024.0.1
 ```
 
 Your Python environment should be ready now. So now you can set up the respective environments according to what you are going to contribute to and build them:

.github/ISSUE_TEMPLATE/BUG.md

@@ -16,7 +16,6 @@ If the bug is related to the setup, please submit the issue at: https://github.c
 - OS (or browser) version:
 - Node.js version:
 - Complete "leon check" (or "npm run check") output:
-- (if using Docker) Complete "npm run docker:check" output:
 - (optional) Leon skill version:
 
 ### Expected Behavior

.github/dependabot.yml

@@ -1,13 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: 'npm'
-    directory: '/'
-    schedule:
-      interval: 'weekly'
-      day: 'friday'
-      time: '22:00'
-    commit-message:
-      prefix: 'chore'
-      include: 'scope'
-    reviewers:
-      - 'louistiti'

.github/workflows/pre-release-nodejs-bridge.yml

@@ -9,19 +9,46 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
+        include:
+          - os: ubuntu-22.04
+            arch: linux-x86_64
+            python_arch: x64
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: lts/*
 
+      - name: Create CI lockfile
+        env:
+          NPM_CONFIG_PACKAGE_LOCK: true
+          PNPM_CONFIG_LOCKFILE: true
+        run: pnpm install --lockfile-only --no-frozen-lockfile
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        shell: bash
+        run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.path }}
+          key: ${{ runner.os }}-pnpm-${{ matrix.arch }}-${{ hashFiles('pnpm-lock.yaml', 'package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-${{ matrix.arch }}-
+
       - name: Set Node.js bridge version
         working-directory: bridges/nodejs/src
         run: |
@@ -32,27 +59,28 @@ jobs:
           echo "Node.js bridge version: ${{ env.NODEJS_BRIDGE_VERSION }}"
 
       - name: Install core
-        run: npm install
+        run: pnpm install --no-frozen-lockfile
 
       - name: Build Node.js bridge
-        run: npm run build:nodejs-bridge
+        run: pnpm run build:nodejs-bridge
 
       - name: Upload Node.js bridge
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: nodejs-bridge-${{ matrix.arch }}
           path: bridges/nodejs/dist/*.zip
 
   draft-release:
     name: Draft-release
     needs: [build]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: lts/*
 
@@ -62,15 +90,31 @@ jobs:
           echo "NODEJS_BRIDGE_VERSION=$(node --require fs --eval "const fs = require('node:fs'); const [, VERSION] = fs.readFileSync('version.ts', 'utf8').split(\"'\"); console.log(VERSION)")" >> $GITHUB_ENV
 
       - name: Download Node.js bridge
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: bridges/nodejs/dist
+          merge-multiple: true
 
-      - uses: marvinpinto/action-automatic-releases@latest
+      - name: Verify Node.js bridge assets
+        shell: bash
+        run: |
+          set -euo pipefail
+          ls -la bridges/nodejs/dist
+          required=(
+            "bridges/nodejs/dist/leon-nodejs-bridge.zip"
+          )
+          for asset in "${required[@]}"; do
+            [ -f "$asset" ] || { echo "Missing asset: $asset"; exit 1; }
+          done
+
+      - name: Create draft release
+        uses: softprops/action-gh-release@v2
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          automatic_release_tag: nodejs-bridge_v${{ env.NODEJS_BRIDGE_VERSION }}
+          tag_name: nodejs-bridge_v${{ env.NODEJS_BRIDGE_VERSION }}
+          name: Node.js Bridge ${{ env.NODEJS_BRIDGE_VERSION }}
           draft: true
           prerelease: false
-          title: Node.js Bridge ${{ env.NODEJS_BRIDGE_VERSION }}
-          files: bridges/nodejs/dist/artifact/*.zip
+          files: bridges/nodejs/dist/*.zip
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pre-release-python-bridge.yml

@@ -13,27 +13,67 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            arch: linux-x86_64
+            python_arch: x64
+          - os: ubuntu-22.04-arm
+            arch: linux-aarch64
+            python_arch: arm64
+          - os: macos-15-intel
+            arch: macosx-x86_64
+            python_arch: x64
+          - os: macos-latest
+            arch: macosx-arm64
+            python_arch: arm64
+          - os: windows-latest
+            arch: win-amd64
+            python_arch: x64
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
+          architecture: ${{ matrix.python_arch }}
 
       - name: Install Pipenv
-        run: pip install --upgrade pip && pip install pipenv==2022.7.24
+        run: pip install --upgrade pip && pip install pipenv==2024.0.1
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: lts/*
 
+      - name: Create CI lockfile
+        env:
+          NPM_CONFIG_PACKAGE_LOCK: true
+          PNPM_CONFIG_LOCKFILE: true
+        run: pnpm install --lockfile-only --no-frozen-lockfile
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        shell: bash
+        run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.path }}
+          key: ${{ runner.os }}-pnpm-${{ matrix.arch }}-${{ hashFiles('pnpm-lock.yaml', 'package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-${{ matrix.arch }}-
+
       - name: Set Python bridge version
         working-directory: bridges/python/src
         run: |
@@ -44,32 +84,33 @@ jobs:
           echo "Python bridge version: ${{ env.PYTHON_BRIDGE_VERSION }}"
 
       - name: Install core
-        run: npm install
+        run: pnpm install --no-frozen-lockfile
 
       - name: Set up Python bridge
-        run: npm run setup:python-bridge
+        run: pnpm run setup:python-bridge
 
       - name: Build Python bridge
-        run: npm run build:python-bridge
+        run: pnpm run build:python-bridge
 
       - name: Upload Python bridge
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: python-bridge-${{ matrix.arch }}
           path: bridges/python/dist/*.zip
 
   draft-release:
     name: Draft-release
     needs: [build]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Set Python bridge version
         working-directory: bridges/python/src
@@ -77,15 +118,35 @@ jobs:
           echo "PYTHON_BRIDGE_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV
 
       - name: Download Python bridge
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: bridges/python/dist
+          merge-multiple: true
 
-      - uses: marvinpinto/action-automatic-releases@latest
+      - name: Verify Python bridge assets
+        shell: bash
+        run: |
+          set -euo pipefail
+          ls -la bridges/python/dist
+          required=(
+            "bridges/python/dist/leon-python-bridge-linux-aarch64.zip"
+            "bridges/python/dist/leon-python-bridge-linux-x86_64.zip"
+            "bridges/python/dist/leon-python-bridge-macosx-arm64.zip"
+            "bridges/python/dist/leon-python-bridge-macosx-x86_64.zip"
+            "bridges/python/dist/leon-python-bridge-win-amd64.zip"
+          )
+          for asset in "${required[@]}"; do
+            [ -f "$asset" ] || { echo "Missing asset: $asset"; exit 1; }
+          done
+
+      - name: Create draft release
+        uses: softprops/action-gh-release@v2
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          automatic_release_tag: python-bridge_v${{ env.PYTHON_BRIDGE_VERSION }}
+          tag_name: python-bridge_v${{ env.PYTHON_BRIDGE_VERSION }}
+          name: Python Bridge ${{ env.PYTHON_BRIDGE_VERSION }}
           draft: true
           prerelease: false
-          title: Python Bridge ${{ env.PYTHON_BRIDGE_VERSION }}
-          files: bridges/python/dist/artifact/*.zip
+          files: bridges/python/dist/*.zip
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pre-release-tcp-server.yml

@@ -13,27 +13,67 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            arch: linux-x86_64
+            python_arch: x64
+          - os: ubuntu-22.04-arm
+            arch: linux-aarch64
+            python_arch: arm64
+          - os: macos-15-intel
+            arch: macosx-x86_64
+            python_arch: x64
+          - os: macos-latest
+            arch: macosx-arm64
+            python_arch: arm64
+          - os: windows-latest
+            arch: win-amd64
+            python_arch: x64
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
+          architecture: ${{ matrix.python_arch }}
 
       - name: Install Pipenv
-        run: pip install --upgrade pip && pip install pipenv==2022.7.24
+        run: pip install --upgrade pip && pip install pipenv==2024.0.1
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: lts/*
 
+      - name: Create CI lockfile
+        env:
+          NPM_CONFIG_PACKAGE_LOCK: true
+          PNPM_CONFIG_LOCKFILE: true
+        run: pnpm install --lockfile-only --no-frozen-lockfile
+
+      - name: Get pnpm store path
+        id: pnpm-store
+        shell: bash
+        run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.path }}
+          key: ${{ runner.os }}-pnpm-${{ matrix.arch }}-${{ hashFiles('pnpm-lock.yaml', 'package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-${{ matrix.arch }}-
+
       - name: Set TCP server version
         working-directory: tcp_server/src
         run: |
@@ -44,32 +84,33 @@ jobs:
           echo "TCP server version: ${{ env.TCP_SERVER_VERSION }}"
 
       - name: Install core
-        run: npm install
+        run: pnpm install --no-frozen-lockfile
 
       - name: Set up TCP server
-        run: npm run setup:tcp-server
+        run: pnpm run setup:tcp-server
 
       - name: Build TCP server
-        run: npm run build:tcp-server
+        run: pnpm run build:tcp-server
 
       - name: Upload TCP server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: tcp-server-${{ matrix.arch }}
           path: tcp_server/dist/*.zip
 
   draft-release:
     name: Draft-release
     needs: [build]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Set TCP server version
         working-directory: tcp_server/src
@@ -77,15 +118,35 @@ jobs:
           echo "TCP_SERVER_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV
 
       - name: Download TCP server
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: tcp_server/dist
+          merge-multiple: true
 
-      - uses: marvinpinto/action-automatic-releases@latest
+      - name: Verify TCP server assets
+        shell: bash
+        run: |
+          set -euo pipefail
+          ls -la tcp_server/dist
+          required=(
+            "tcp_server/dist/leon-tcp-server-linux-aarch64.zip"
+            "tcp_server/dist/leon-tcp-server-linux-x86_64.zip"
+            "tcp_server/dist/leon-tcp-server-macosx-arm64.zip"
+            "tcp_server/dist/leon-tcp-server-macosx-x86_64.zip"
+            "tcp_server/dist/leon-tcp-server-win-amd64.zip"
+          )
+          for asset in "${required[@]}"; do
+            [ -f "$asset" ] || { echo "Missing asset: $asset"; exit 1; }
+          done
+
+      - name: Create draft release
+        uses: softprops/action-gh-release@v2
         with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          automatic_release_tag: tcp-server_v${{ env.TCP_SERVER_VERSION }}
+          tag_name: tcp-server_v${{ env.TCP_SERVER_VERSION }}
+          name: TCP Server ${{ env.TCP_SERVER_VERSION }}
           draft: true
           prerelease: false
-          title: TCP Server ${{ env.TCP_SERVER_VERSION }}
-          files: tcp_server/dist/artifact/*.zip
+          files: tcp_server/dist/*.zip
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -12,6 +12,9 @@ logs/*
 core/config/**/*.json
 bin/coqui/*
 bin/flite/*
+bin/nvidia/*
+bin/pytorch/torch/*
+scripts/out/*.md
 
 package-lock.json
 *.pyc
@@ -23,14 +26,23 @@ debug.log
 .last-skill-npm-install
 leon.json
 bridges/python/src/Pipfile.lock
+bridges/toolkits/**/bins
 tcp_server/src/Pipfile.lock
 !tcp_server/**/.gitkeep
+!bridges/toolkits/**/.gitkeep
 !bridges/python/**/.gitkeep
 !bridges/nodejs/**/.gitkeep
 !**/*.sample*
 skills/**/src/settings.json
 skills/**/memory/*.json
+bridges/toolkits/**/settings.json
 core/data/models/*.nlp
+core/data/models/*.json
+core/data/models/llm/*
+core/data/models/audio/tts/**/*.*
+!core/data/models/audio/tts/config.json
+core/data/models/audio/asr/**/*.*
+!core/data/models/**/.gitkeep
 package.json.backup
 .python-version
 schemas/**/*.json

.husky/commit-msg

@@ -1,11 +1,8 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
 if ! [ -x "$(command -v npm)" ]; then
   echo "npm: command not found"
-  echo "If you use a version manager tool such as nvm and a git GUI such as GitKraken, please read: https://typicode.github.io/husky/#/?id=command-not-found" >&2
+  echo "If you use a version manager tool such as nvm and a git GUI such as GitKraken, please read: https://typicode.github.io/husky/how-to.html#node-version-managers-and-guis" >&2
   exit 1
 else
-  npx ts-node scripts/commit-msg.js
+  npx tsx scripts/commit-msg.js
 fi
 

.husky/pre-commit

@@ -1,4 +1,7 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
-npx lint-staged
+if ! [ -x "$(command -v npm)" ]; then
+  echo "npm: command not found"
+  echo "If you use a version manager tool such as nvm and a git GUI such as GitKraken, please read: https://typicode.github.io/husky/how-to.html#node-version-managers-and-guis" >&2
+  exit 1
+else
+  npm run pre-commit
+fi

.prettierignore

@@ -0,0 +1 @@
+README.md

Dockerfile

@@ -1,60 +0,0 @@
-FROM ubuntu:20.04
-ENV IS_DOCKER true
-
-# Replace shell with bash so we can source files
-RUN rm /bin/sh && ln -s /bin/bash /bin/sh
-
-# Set debconf to run non-interactively
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-# Install base dependencies
-RUN apt-get update && apt-get install --yes -q --no-install-recommends \
-  apt-transport-https \
-  build-essential \
-  ca-certificates \
-  curl \
-  git \
-  wget \
-  libssl-dev \
-  openssl \
-  libz-dev \
-  zlib1g-dev \
-  libbz2-dev \
-  libreadline-dev \
-  libsqlite3-dev \
-  llvm \
-  libncurses5-dev \
-  xz-utils \
-  tk-dev libxml2-dev \
-  libxmlsec1-dev \
-  libffi-dev \
-  liblzma-dev \
-  libgdbm-dev \
-  libnss3-dev \
-  libc6-dev
-
-# Run the container as an unprivileged user
-RUN groupadd docker && useradd -g docker -s /bin/bash -m docker
-USER docker
-WORKDIR /home/docker
-
-# Install Node.js with nvm
-ENV NVM_DIR /home/docker/.nvm
-ENV NODE_VERSION v16.18.0
-
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
-RUN /bin/bash -c "source $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use --delete-prefix $NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules
-ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH
-
-# Install Leon
-WORKDIR /home/docker/leon
-USER root
-RUN chown -R docker /home/docker/leon
-USER docker
-COPY --chown=docker ./ ./
-RUN npm install
-RUN npm run build
-
-CMD ["npm", "start"]

README.md

@@ -17,7 +17,7 @@ _<p align="center">Your open-source personal assistant.</p>_
   <a href="https://github.com/leon-ai/leon/actions/workflows/tests.yml"><img src="https://github.com/leon-ai/leon/actions/workflows/tests.yml/badge.svg?branch=develop" /></a>
   <a href="https://github.com/leon-ai/leon/actions/workflows/lint.yml"><img src="https://github.com/leon-ai/leon/actions/workflows/lint.yml/badge.svg?branch=develop" /></a>
   <br>
-  <a href="https://discord.gg/MNQqqKg"><img src="https://svgshare.com/i/V09.svg"/></a>
+  <a href="https://discord.gg/MNQqqKg"><img src="https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white" /></a>
 </p>
 
 <p align="center">
@@ -30,38 +30,65 @@ _<p align="center">Your open-source personal assistant.</p>_
 
 ---
 
-## Current State
+## Important Notice (as of 2026-01-11)
+
+> [!IMPORTANT]
+> **Leon is currently undergoing a massive architectural rewrite.**
+>
+> The `develop` branch is highly experimental and may be unstable as I implement the new agentic core.
+> 
+> - If you are looking for the legacy, stable version (pre-LLM), please use the `master` branch.
+> - If you want to contribute to the future of Leon (LLMs, Agents, Automation), you are in the right place.
+
+### Outdated Documentation
+
+Please note that the current documentation and this README are outdated regarding the technical architecture. We are moving away from simple classification toward a hybrid approach involving Local LLMs, Transformers, and Atomic Tools. Updated documentation will be released alongside the new core stability.
+
+### Project Evolution and Future Plans
+
+**I have been working on Leon since 2017**. While development has been inconsistent in the past, the current era of AI unlocks capabilities that were previously impossible. I'm now transitioning Leon from a standard assistant to a fully **autonomous personal AI assistant** designed to be used by technical hobbyists to non-tech users.
+
+I'm currently building the foundation for the next generation of Leon, focusing on 3 key milestones:
+
+**1. Workflow Architecture and Atomic Tools**
+
+We are restructuring Leon around a robust flow: `Skills > Actions > Tools > Functions (> Binaries)`.
+Instead of monolithic scripts, Leon will use atomic components (e.g. compiled binaries using ONNX runtime) to execute complex workflows.
+
+- Example: a "Video Translator" skill won't just be a script; it will be a workflow where Leon orchestrates tools like vocal isolation, zero-shot voice cloning, ASR, audio gender recognition, etc. to achieve the result.
+
+**2. Autonomous Skill Generation (self-coding)**
+
+We are developing a meta-skill capable of writing code for new skills automatically.
+
+- Leon will analyze a request, check if a skill exists, and if not, write the code itself following our strict architectural standards.
+- It will leverage existing tools and inject the new skill directly into its memory for future reuse.
+
+**3. Agentic Behavior (ReAct) and Local LLM Optimization**
+
+The ultimate phase will be to adopt the ReAct (Reason + Act) approach.
+
+- Leon will be provided with low-level **tools** (organized in toolkits, e.g., `music_audio` containing FFmpeg).
+- Using Local LLMs, Leon will loop through thoughts and actions to solve problems dynamically.
+- Optimization: we are implementing strict context filtering to save tokens, reduce hallucinations, and ensure high performance on local hardware.
+
+**Get Involved**
+
+[Join us on Discord](https://discord.gg/MNQqqKg) to ask questions, or express interest in becoming an active contributor.
+
+- Check out [the roadmap](http://roadmap.getleon.ai/) for more information on our upcoming plans.
+- Watch a [preview of our last progress](https://www.youtube.com/watch?v=6CInSt6pTVA) to see what we've been working on.
+
+---
 
 ### Why is there a small amount of contributors?
 
-I'm taking a lot of time to work on the new core of Leon due to personal reasons. I can only work on it after work and on weekends. Hence, **I'm blocking any potential contribution as the whole core of Leon is coming with many breaking changes**. Many of you are willing to contribute in Leon (create new skills, help to improve the core, translations and so on...), a big thanks to every one of you!
+I'm taking a lot of time to work on the new core of Leon due to personal reasons. I can only work on it during my spare time. Hence, I'm blocking any contribution as the whole core of Leon is coming with many breaking changes. Many of you are willing to contribute in Leon (create new skills, help to improve the core, translations and so on...), a big thanks to every one of you!
 
 While I would love to devote more time to Leon, I'm currently unable to do so because I have bills to pay. I have some ideas about how to monetize Leon in the future (Leon's core will always remain open source), but before to get there there is still a long way to go.
 
 Until then, any financial support by [sponsoring Leon](http://sponsor.getleon.ai) is much appreciated 🙂
 
-### How about large language models and Leon?
-
-Since AI gained in popularity and large language models are getting more and more traction, many of you joined our community. A huge welcome to all of you! 🤗
-
-At the moment, Leon's NLU will remain intents first with his own model without relying on an LLM. It is important that Leon can run 100% offline and I'm confident that with the downsizing techniques such as quantization Leon will sooner or later work with LLMs at his core and still be able to run on edge.
-
-Here is how LLMs may help Leon in the future:
-
-- Intent fallback: when an utterance cannot match an intent, then rely on an LLM to provide results.
-- New named entity recognition engine: provide a better solution to extract entities from utterances such as fruits, numbers, cities, durations, persons, etc.
-- Skill features: let skills leverage LLMs to provide out-of-the-box NLP features such as summarization, knowledge base, translation, sentiment analysis and so on...
-- Skill building: LLMs can help to develop skills such as paraphrasing utterance samples, translate answers, convert code from our Python bridge to the upcoming JavaScript bridge and vice versa, etc.
-- More...
-
-### What's Next?
-
-Once the new core released, we'll work on the community aspect of Leon. For example, better organize [our Discord](https://discord.gg/MNQqqKg), planify regular calls, work on skills together, etc. It is very important for Leon to have a real community. At that moment, the skills platform will already be online, so it'll be easier to sync our progress and publish new skills.
-
-- Feel free to check out the Git development branches and our [next major milestones](https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/#whats-next).
-- And the [detailed roadmap](http://roadmap.getleon.ai).
-- Many exciting things are coming up, hence no new documentation and test are going to be written until the official release of Leon.
-
 ---
 
 ## Latest Release
@@ -122,8 +149,8 @@ Gitpod will automatically set up an environment and run an instance for you.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/) >= 16
-- [npm](https://npmjs.com/) >= 8
+- [Node.js](https://nodejs.org/) >= 22.13.1
+- [npm](https://npmjs.com/) >= 10.9.2
 - Supported OSes: Linux, macOS and Windows
 
 To install these prerequisites, you can follow the [How To section](https://docs.getleon.ai/how-to/) of the documentation.
@@ -152,23 +179,16 @@ leon start
 # Hooray! Leon is running
 ```
 
-### Docker Installation
-
-```sh
-# Install Leon
-leon create birth --docker
-
-# Run
-leon start
-
-# Go to http://localhost:1337
-# Hooray! Leon is running
-```
-
 ## 📚 Documentation
 
 For full documentation, visit [docs.getleon.ai](https://docs.getleon.ai).
 
+## 🇫🇷 Documenting the Journey on YouTube
+
+[I'm documenting the journey on YouTube](https://www.youtube.com/@louisgyt) in developing our dear Leon. I also take you along in my daily life here in China.
+
+For non-French speakers, translated English subtitles are available.
+
 ## 📺 Video
 
 [Watch a demo](https://www.youtube.com/watch?v=p7GRGiicO1c).

app/src/css/style.scss

@@ -1,4 +1,11 @@
-@import url(https://fonts.googleapis.com/css?family=Open+Sans:400,600,700,800);
+@import '@fontsource/source-sans-pro/200.css';
+@import '@fontsource/source-sans-pro/300.css';
+@import '@fontsource/source-sans-pro/400.css';
+@import '@fontsource/source-sans-pro/600.css';
+@import '@fontsource/source-sans-pro/700.css';
+@import '@fontsource/source-sans-pro/900.css';
+@import 'remixicon/fonts/remixicon.css';
+@import 'voice-energy/main.scss';
 
 html,
 body,
@@ -83,7 +90,6 @@ video {
   padding: 0;
   border: 0;
   font-size: 100%;
-  font: inherit;
   vertical-align: baseline;
 }
 
@@ -129,48 +135,68 @@ table {
   --light-black-color: #222426;
   --white-color: #fff;
   --grey-color: #323739;
+  --blue-color: #1c75db;
+  --pink-color: #ed297a;
+
+  .settingup {
+    --a-loader-size-md: 20px !important;
+  }
 }
 
 a {
   color: inherit;
 }
 
-ul li {
+#feed ul li:not(.aurora-list-item) {
   margin-left: 20px;
 }
 
 body {
   color: var(--white-color);
   background-color: var(--black-color);
-  font-family: 'Open Sans', sans-serif;
+  font-family:
+    'Source Sans Pro',
+    system-ui,
+    -apple-system,
+    BlinkMacSystemFont,
+    'Segoe UI',
+    Roboto,
+    'Helvetica Neue',
+    Arial,
+    'Noto Sans',
+    sans-serif,
+    'Apple Color Emoji',
+    'Segoe UI Emoji',
+    'Segoe UI Symbol',
+    'Noto Color Emoji';
   font-weight: 400;
 }
 body > * {
   transition: opacity 0.5s;
 }
-body.settingup > * {
+body.settingup > *:not(#init) {
   opacity: 0;
 }
-body.settingup::after {
-  position: absolute;
-  content: '';
-  width: 32px;
-  height: 32px;
-  background-color: #777;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
-  border-radius: 50%;
-  animation: scaleout 0.6s infinite ease-in-out;
+#init .not-initialized {
+  visibility: hidden;
 }
-@keyframes scaleout {
-  0% {
-    transform: scale(0);
-  }
-  100% {
-    transform: scale(1);
-    opacity: 0;
-  }
+#init .initialized {
+  opacity: 0;
+  visibility: hidden;
+}
+
+kbd {
+  font-family: 'Source Sans Pro', monospace;
+  display: inline-block;
+  background-color: var(--light-black-color);
+  color: rgba(255, 255, 255, 0.4);
+  border-radius: 4px;
+  text-align: center;
+  min-width: 16px;
+  min-height: 16px;
+  line-height: 16px !important;
+  padding: 2px 6px !important;
+  margin: 0 !important;
 }
 
 main {
@@ -187,24 +213,43 @@ footer {
   text-align: center;
   left: 50%;
   bottom: 0;
+  line-height: 18px;
   transform: translate(-50%, -50%);
 }
 
-input {
+textarea {
+  font-family: inherit;
   text-align: center;
   color: var(--white-color);
   width: 100%;
   border: none;
   border-bottom: 2px solid var(--grey-color);
   background: none;
-  font-weight: 400;
+  font-weight: 600;
   font-size: 4em;
   padding-right: 39px;
+  height: 140px;
+  resize: none;
+  overflow-y: auto;
+}
+textarea::-webkit-scrollbar {
+  width: 6px;
+}
+textarea::-webkit-scrollbar-thumb {
+  background-color: rgba(255, 255, 255, 0.2);
+  border-radius: 12px;
 }
 
-small {
+#tip {
+  display: inline-flex;
+  margin-top: 2px;
   color: var(--white-color);
-  font-size: 0.7em;
+  line-height: 22px;
+  font-size: 0.9em;
+  gap: 20px;
+  li {
+    margin-left: 0;
+  }
 }
 
 .hide {
@@ -218,10 +263,32 @@ small {
   height: 40px;
 }
 
+#top-container {
+  position: absolute;
+  top: 4%;
+  color: var(--grey-color);
+  display: flex;
+  width: 100%;
+  justify-content: space-between;
+}
+#mood {
+  position: relative;
+  font-size: 16px;
+}
+#info {
+  position: relative;
+  text-decoration: underline;
+  background: none;
+  border: none;
+  color: var(--grey-color);
+  cursor: pointer;
+  font-size: inherit;
+}
+
 #feed {
   position: absolute;
   width: 100%;
-  top: 10%;
+  top: 8%;
   height: 50%;
   overflow-y: auto;
   border: 2px solid var(--grey-color);
@@ -278,6 +345,15 @@ small {
   }
 }
 
+.llm-token {
+  opacity: 0;
+  transition: opacity 2.5s;
+}
+
+.llm-token.fade-in {
+  animation: fadeIn 2.5s forwards;
+}
+
 .bubble-container {
   padding: 6px;
 }
@@ -300,7 +376,7 @@ small {
 }
 
 .bubble {
-  padding: 6px 12px;
+  padding: 10px 16px;
   border-radius: 16px;
   display: inline-block;
   max-width: 60%;
@@ -309,6 +385,8 @@ small {
   opacity: 0;
   animation: fadeIn 0.2s ease-in forwards;
   overflow: hidden;
+  font-size: 1.8rem;
+  line-height: 2.4rem;
 }
 #feed .me .bubble {
   background-color: #1c75db;
@@ -343,7 +421,9 @@ small {
   padding: 2px 8px;
   font-size: inherit;
   cursor: pointer;
-  transition: background-color 0.2s, color 0.2s;
+  transition:
+    background-color 0.2s,
+    color 0.2s;
 }
 .suggestion:hover {
   color: var(--black-color);
@@ -353,7 +433,7 @@ small {
 #input-container {
   position: absolute;
   width: 100%;
-  bottom: 22%;
+  bottom: 18%;
 }
 
 #mic-container {
@@ -413,3 +493,116 @@ small {
     transform: scale(1);
   }
 }
+
+/* Clickable URL styles */
+.clickable-url {
+  text-decoration: underline;
+}
+
+/* Clickable file path styles */
+.clickable-path {
+  cursor: pointer;
+  text-decoration: underline;
+}
+
+/* Tool Output Container Styles */
+.tool-group-container {
+  margin: 8px 6px;
+  border: 1px solid var(--grey-color);
+  border-radius: 8px;
+  background-color: #1a1a1a;
+  font-family: 'Courier New', Consolas, monospace;
+  font-size: 0.9em;
+  opacity: 0;
+  animation: fadeIn 0.3s ease-in forwards;
+}
+
+.tool-header {
+  display: flex;
+  align-items: center;
+  padding: 8px 12px;
+  background-color: var(--light-black-color);
+  border-radius: 8px 8px 0 0;
+  cursor: pointer;
+  border-bottom: 1px solid var(--grey-color);
+  transition: background-color 0.2s;
+}
+
+.tool-header:hover {
+  background-color: #2a2c2e;
+}
+
+.tool-icon {
+  color: var(--blue-color);
+  margin-right: 8px;
+  font-size: 16px;
+}
+
+.tool-name {
+  flex: 1;
+  font-weight: 600;
+  color: var(--white-color);
+  font-size: 0.95em;
+}
+
+.expand-icon {
+  color: var(--grey-color);
+  font-size: 18px;
+  transition: transform 0.2s ease;
+}
+
+.expand-icon.rotated {
+  transform: rotate(180deg);
+}
+
+.tool-content {
+  max-height: 0;
+  overflow: hidden;
+  transition: max-height 0.3s ease;
+}
+
+.tool-content.expanded {
+  max-height: 500px;
+  overflow-y: auto;
+}
+
+.tool-content::-webkit-scrollbar {
+  width: 4px;
+}
+
+.tool-content::-webkit-scrollbar-thumb {
+  background-color: rgba(255, 255, 255, 0.1);
+  border-radius: 2px;
+}
+
+.shell-output {
+  padding: 12px;
+  background-color: #0d1117;
+  border-radius: 0 0 8px 8px;
+  min-height: 40px;
+}
+
+.shell-message {
+  margin: 2px 0;
+  line-height: 1.4;
+  color: #e6edf3;
+  word-break: break-word;
+}
+
+.shell-prompt {
+  color: var(--pink-color);
+  font-weight: bold;
+  margin-right: 8px;
+}
+
+.shell-message .clickable-path {
+  color: var(--blue-color);
+  background-color: rgba(28, 117, 219, 0.1);
+  padding: 1px 4px;
+  border-radius: 3px;
+  border: 1px solid rgba(28, 117, 219, 0.3);
+}
+
+.shell-message .clickable-url {
+  color: var(--blue-color);
+}

app/src/css/voice-energy/base.scss

@@ -0,0 +1,162 @@
+/**
+ * Overlay and containers
+ */
+
+body.voice-mode-enabled {
+  #voice-overlay-transitor,
+  #voice-overlay-bg {
+    visibility: visible;
+  }
+  #voice-overlay-bg {
+    opacity: 1;
+  }
+}
+#voice-overlay-transitor {
+  position: fixed;
+  background-color: var(--black-color);
+  z-index: 10;
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  will-change: transform;
+  animation: scaleIn 1s;
+}
+@keyframes scaleIn {
+  0% {
+    transform: scale3d(0, 0, 1);
+  }
+  100% {
+    transform: scale3d(172, 172, 1);
+  }
+}
+
+#voice-status,
+#voice-tips {
+  color: var(--grey-color);
+  text-align: center;
+}
+#voice-status {
+  font-size: 17px;
+  font-style: italic;
+}
+#voice-tips {
+  margin-top: 32px;
+  line-height: 18px;
+  font-size: 15px;
+}
+
+#voice-overlay-bg {
+  visibility: hidden;
+  cursor: pointer;
+  opacity: 0;
+  position: fixed;
+  width: 100vw;
+  height: 100vh;
+  z-index: 100;
+  will-change: opacity;
+  display: flex;
+  justify-content: center;
+  // backdrop-filter: saturate(140%) blur(5px);
+  /*background-color: rgba(0, 0, 0, .9);*/
+  background-color: var(--black-color);
+}
+@keyframes skipFadeIn {
+  0% {
+    opacity: 1;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
+#voice-container {
+  opacity: 0;
+  position: relative;
+  top: 64px;
+  display: flex;
+  flex-direction: column;
+  width: 1024px;
+  height: 756px;
+  align-items: center;
+  gap: 64px;
+  animation: fadeIn 1s 3s both;
+}
+
+#voice-energy-container {
+  --neon-size: 228px;
+
+  // animation: fadeIn 1s 1.5s both;
+  //opacity: 0;
+  overflow: hidden;
+  position: relative;
+  height: 400px;
+  border-radius: 50%;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+p#voice-speech {
+  width: 100%;
+  height: 100%;
+  flex: 1;
+  text-align: center;
+  font-size: 3rem;
+  font-weight: 600;
+}
+
+/**
+ * Neons
+ */
+
+.voice-neon {
+  position: absolute;
+  z-index: 10;
+  width: var(--neon-size);
+  height: var(--neon-size);
+}
+
+#purple-neon-blur {
+  --neon-blur: calc(var(--neon-size) + 96px);
+
+  position: absolute;
+  z-index: 0;
+  opacity: 0.7;
+  width: var(--neon-blur);
+  height: var(--neon-blur);
+}
+
+#blue-neon-1 {
+  margin-top: -8px;
+  margin-left: 12px;
+}
+
+#blue-neon-2 {
+  margin-top: 8px;
+  margin-right: 12px;
+}
+
+/**
+ * Particles
+ */
+
+.voice-particle {
+  position: absolute;
+  width: 3px;
+  height: 3px;
+  border-radius: 50%;
+  opacity: 1;
+  will-change: transform, opacity;
+  animation-duration: 1s;
+  animation-iteration-count: infinite;
+}
+.voice-particle.blue {
+  background-color: #c4e0ff;
+  box-shadow: 0 0 2px 2px var(--blue-color);
+}
+.voice-particle.pink {
+  background-color: #ffb9d7;
+  box-shadow: 0 0 2px 2px var(--pink-color);
+}

app/src/css/voice-energy/idle.scss

@@ -0,0 +1,59 @@
+/**
+ * IDLE status
+ */
+
+#voice-energy-container.idle {
+  .voice-particle {
+    visibility: hidden;
+    animation-play-state: paused;
+    opacity: 0;
+  }
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    transform: scale(1);
+  }
+  #purple-neon-blur circle {
+    filter: drop-shadow(0px 0px 64px mix(#ed297a, #1c75db));
+    animation: idleNeonBlurBreath 2.2s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: idleBouncePinkNeon1 1.8s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: idleMoveBlueNeon1 1.8s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: idleMoveBlueNeon2 1.8s 0.5s infinite alternate;
+  }
+}
+
+@keyframes idleNeonBlurBreath {
+  100% {
+    filter: drop-shadow(0px 0px 0px mix(#ed297a, #1c75db));
+  }
+}
+@keyframes idleBouncePurpleNeonBlur {
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes idleBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes idleMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes idleMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}

app/src/css/voice-energy/listening.scss

@@ -0,0 +1,59 @@
+/**
+ * Listening status
+ */
+
+#voice-energy-container.listening {
+  .voice-particle {
+    visibility: hidden;
+    animation-play-state: paused;
+    opacity: 0;
+  }
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    transform: scale(1);
+  }
+  #purple-neon-blur circle {
+    filter: drop-shadow(0px 0px 64px mix(#ed297a, #1c75db));
+    animation: listeningNeonBlurBreath 0.7s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: listeningBouncePinkNeon1 0.3s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: listeningMoveBlueNeon1 0.3s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: listeningMoveBlueNeon2 0.3s 0.5s infinite alternate;
+  }
+}
+
+@keyframes listeningNeonBlurBreath {
+  100% {
+    filter: drop-shadow(0px 0px 0px mix(#ed297a, #1c75db));
+  }
+}
+@keyframes listeningBouncePurpleNeonBlur {
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes listeningBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes listeningMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes listeningMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}

app/src/css/voice-energy/main.scss

@@ -0,0 +1,5 @@
+@import 'base.scss';
+@import 'listening.scss';
+@import 'idle.scss';
+@import 'processing.scss';
+@import 'talking.scss';

app/src/css/voice-energy/processing.scss

@@ -0,0 +1,88 @@
+@use 'sass:math';
+
+/**
+ * Processing status
+ */
+
+#voice-energy-container.processing {
+  #purple-neon-blur {
+    animation: processingBouncePurpleNeonBlur 1s infinite alternate;
+  }
+  #pink-neon-1 {
+    animation: processingBouncePinkNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-1 {
+    animation: processingMoveBlueNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-2 {
+    animation: processingMoveBlueNeon2 0.5s infinite alternate;
+  }
+}
+
+@keyframes processingBouncePurpleNeonBlur {
+  50% {
+    transform: scale(1.07);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes processingBouncePinkNeon1 {
+  50% {
+    transform: scale(1.02);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes processingMoveBlueNeon1 {
+  0% {
+    transform: translateX(0) translateY(0);
+  }
+  33% {
+    transform: translateY(-3px) translateX(-2px);
+  }
+  66% {
+    transform: translateY(-3px) translateX(3px);
+  }
+  100% {
+    transform: translateY(-3px) translateX(1px);
+  }
+}
+@keyframes processingMoveBlueNeon2 {
+  0% {
+    transform: translateX(0) translateY(0);
+  }
+  33% {
+    transform: translateY(3px) translateX(2px);
+  }
+  66% {
+    transform: translateY(3px) translateX(3px);
+  }
+  100% {
+    transform: translateY(3px) translateX(-1px);
+  }
+}
+
+@for $i from 0 through 31 {
+  .processing .voice-particle[data-particle='#{$i}'] {
+    animation-delay: #{$i * 0.1}s;
+  }
+
+  .processing .voice-particle[data-particle='#{$i}'] {
+    animation-name: processingMoveParticle#{$i};
+  }
+
+  #voice-energy-container.processing {
+    @keyframes processingMoveParticle#{$i} {
+      75% {
+        opacity: 0.1;
+      }
+      100% {
+        opacity: 1;
+        transform: translateX(math.cos(11.25deg * $i) * 110px)
+          translateY(math.sin(11.25deg * $i) * 110px);
+      }
+    }
+  }
+}

app/src/css/voice-energy/talking.scss

@@ -0,0 +1,82 @@
+@use 'sass:math';
+
+/**
+ * Talking status
+ */
+
+#voice-energy-container.talking {
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    animation: talkingBouncePurpleNeonBlur 1s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: talkingBouncePinkNeon1 0.5s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: talkingMoveBlueNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: talkingMoveBlueNeon2 0.5s 0.3s infinite alternate;
+  }
+}
+
+@keyframes talkingBouncePurpleNeonBlur {
+  50% {
+    transform: scale(1.07);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes talkingBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes talkingMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes talkingMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}
+
+.talking .voice-particle {
+  opacity: 0;
+  animation-duration: 2s;
+}
+@for $i from 0 through 31 {
+  .talking .voice-particle[data-particle='#{$i}'] {
+    animation-delay: #{$i * 0.2}s;
+    // animation-duration: #{$i * 0.5}s;
+  }
+
+  .talking .voice-particle[data-particle='#{$i}'] {
+    animation-name: talkingMoveParticle#{$i};
+  }
+
+  #voice-energy-container.talking {
+    @keyframes talkingMoveParticle#{$i} {
+      0% {
+        opacity: 1;
+        transform: translate(0);
+      }
+      50% {
+        opacity: 0;
+      }
+      100% {
+        opacity: 0;
+        transform: translateX(math.cos(math.random() * 360deg))
+          translateY(math.sin(math.random() * 360deg));
+      }
+    }
+  }
+}

app/src/custom-aurora-components/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/timer.tsx

@@ -0,0 +1,70 @@
+import React, { useState, useEffect } from 'react'
+import { CircularProgress, Flexbox, Text } from '@leon-ai/aurora'
+
+interface TimerProps {
+  initialTime: number
+  interval: number
+  totalTimeContent: string
+  initialProgress?: number
+  onEnd?: () => void
+}
+
+function formatTime(seconds: number): string {
+  const minutes = seconds >= 60 ? Math.floor(seconds / 60) : 0
+  const remainingSeconds = seconds % 60
+  const formattedMinutes = minutes < 10 ? `0${minutes}` : minutes
+  const formattedSeconds =
+    remainingSeconds < 10 ? `0${remainingSeconds}` : remainingSeconds
+
+  return `${formattedMinutes}:${formattedSeconds}`
+}
+
+export function Timer({
+  initialTime,
+  initialProgress,
+  interval,
+  totalTimeContent,
+  onEnd
+}: TimerProps) {
+  const [progress, setProgress] = useState(initialProgress || 0)
+  const [timeLeft, setTimeLeft] = useState(initialTime)
+
+  useEffect(() => {
+    setTimeLeft(initialTime)
+    setProgress(progress)
+  }, [initialTime])
+
+  useEffect(() => {
+    if (timeLeft <= 0) {
+      return
+    }
+
+    const timer = setInterval(() => {
+      setTimeLeft((prevTime) => {
+        const newTime = prevTime - 1
+
+        if (newTime <= 0 && onEnd) {
+          onEnd()
+        }
+
+        return newTime
+      })
+      setProgress((prevProgress) => prevProgress + 100 / initialTime)
+    }, interval)
+
+    return () => clearInterval(timer)
+  }, [initialTime, interval, timeLeft])
+
+  return (
+    <CircularProgress value={progress} size="lg">
+      <Flexbox gap="xs" alignItems="center" justifyContent="center">
+        <Text fontSize="lg" fontWeight="semi-bold">
+          {formatTime(timeLeft)}
+        </Text>
+        <Text fontSize="xs" secondary>
+          {totalTimeContent}
+        </Text>
+      </Flexbox>
+    </CircularProgress>
+  )
+}

app/src/index.html

@@ -1,15 +1,306 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <link rel="stylesheet" href="/css/style.css" />
     <link rel="icon" type="image/png" href="/img/favicon.png" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Leon</title>
+    <style>
+      @import './css/style.scss';
+    </style>
   </head>
   <body class="settingup">
+    <div id="init"></div>
+    <div id="voice-overlay-bg">
+      <div id="voice-container">
+        <div>
+          <div id="voice-energy-container">
+            <svg
+              class="voice-neon"
+              id="pink-neon-1"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_16)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="113.5"
+                  stroke="#FFF"
+                  stroke-width="10"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_16"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="1"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_16"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.929412 0 0 0 0 0.160784 0 0 0 0 0.478431 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_16"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              class="voice-neon"
+              id="blue-neon-1"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_24)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="116.5"
+                  stroke="#FFF"
+                  stroke-width="4"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_24"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="1"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_24"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.109804 0 0 0 0 0.458824 0 0 0 0 0.858824 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_24"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              class="voice-neon"
+              id="blue-neon-2"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_24)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="116.5"
+                  stroke="#FFF"
+                  stroke-width="4"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_24"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="5"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_24"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.109804 0 0 0 0 0.458824 0 0 0 0 0.858824 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_24"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              id="purple-neon-blur"
+              viewBox="0 0 341 341"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g opacity="0.72" filter="url(#filter0_f_18_61)">
+                <circle
+                  cx="170.5"
+                  cy="170.5"
+                  r="116.5"
+                  stroke="#ED297A"
+                  stroke-width="24"
+                />
+              </g>
+              <g opacity="0.72" filter="url(#filter1_f_18_61)">
+                <circle
+                  cx="170.5"
+                  cy="170.5"
+                  r="116.5"
+                  stroke="#1C75DB"
+                  stroke-width="24"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_f_18_61"
+                  x="0"
+                  y="0"
+                  width="341"
+                  height="341"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feGaussianBlur
+                    stdDeviation="21"
+                    result="effect1_foregroundBlur_18_61"
+                  />
+                </filter>
+                <filter
+                  id="filter1_f_18_61"
+                  x="0"
+                  y="0"
+                  width="341"
+                  height="341"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feGaussianBlur
+                    stdDeviation="21"
+                    result="effect1_foregroundBlur_18_61"
+                  />
+                </filter>
+              </defs>
+            </svg>
+          </div>
+          <div id="voice-status"></div>
+          <div id="voice-tips">
+            It is recommended to use a headset for a better voice experience.
+            <br />
+            Otherwise, if your microphone is too sensitive or speakers are too
+            loud,
+            <br />
+            Leon may hear his own voice and get confused.
+          </div>
+        </div>
+        <p id="voice-speech"></p>
+      </div>
+    </div>
     <main>
-      <div id="root"></div>
+      <div id="top-container">
+        <div id="mood"></div>
+        <button id="info">Info</button>
+      </div>
       <div id="feed">
         <p id="no-bubble" class="hide">
           You can start to interact with Leon, don't be shy.
@@ -27,16 +318,20 @@
           <div id="sonar"></div>
         </div>
         <label for="utterance"></label>
-        <input type="text" id="utterance" autocomplete="off" autofocus />
-        <small>
-          Use <kbd>↑</kbd> <kbd>↓</kbd> to browse history; <kbd>↵</kbd> to
-          submit;
-          <kbd>alt + c to listen.</kbd>
-        </small>
+        <textarea id="utterance" autocomplete="off" autofocus></textarea>
+        <ul id="tip">
+          <li><kbd>enter</kbd> to submit.</li>
+          <li><kbd>shift</kbd> + <kbd>enter</kbd> for new line.</li>
+          <li>
+            <kbd>shift</kbd> + <kbd>↑</kbd> / <kbd>↓</kbd> to browse history.
+          </li>
+          <li><kbd>alt</kbd> / <kbd>cmd</kbd> + <kbd>c</kbd> to listen.</li>
+        </ul>
       </div>
     </main>
     <footer>
       <div id="logo"></div>
+      <br />
       <div id="version">
         <small>v</small>
       </div>

app/src/js/app.tsx

@@ -1,9 +0,0 @@
-function App() {
-  return (
-    <>
-      <p>Hello from React</p>
-    </>
-  )
-}
-
-export default App

app/src/js/aurora/button.tsx

@@ -1,9 +0,0 @@
-import type React from 'react'
-
-interface Props {
-  children: React.ReactNode
-}
-
-export function Button({ children }: Props) {
-  return <button>{children}</button>
-}

app/src/js/chatbot.js

@@ -1,13 +1,33 @@
-const MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE = 340
+import { createElement } from 'react'
+import { createRoot } from 'react-dom/client'
+import axios from 'axios'
+// eslint-disable-next-line no-redeclare
+import { WidgetWrapper, Flexbox, Loader, Text } from '@leon-ai/aurora'
+
+import renderAuroraComponent from './render-aurora-component'
+import ToolUIHandler from './tool-ui-handler'
+
+const WIDGETS_TO_FETCH = []
+const WIDGETS_FETCH_CACHE = new Map()
+const REPLACED_MESSAGES = new Set()
 
 export default class Chatbot {
-  constructor() {
+  constructor(socket, serverURL) {
+    this.socket = socket
+    this.serverURL = serverURL
     this.et = new EventTarget()
     this.feed = document.querySelector('#feed')
     this.typing = document.querySelector('#is-typing')
     this.noBubbleMessage = document.querySelector('#no-bubble')
     this.bubbles = localStorage.getItem('bubbles')
     this.parsedBubbles = JSON.parse(this.bubbles)
+
+    // Initialize tool UI handler
+    this.toolUIHandler = new ToolUIHandler(
+      this.feed,
+      this.scrollDown.bind(this),
+      this.formatMessage.bind(this)
+    )
   }
 
   async init() {
@@ -15,11 +35,27 @@ export default class Chatbot {
     this.scrollDown()
 
     this.et.addEventListener('to-leon', (event) => {
-      this.createBubble('me', event.detail)
+      this.createBubble({
+        who: 'me',
+        string: event.detail
+      })
     })
 
     this.et.addEventListener('me-received', (event) => {
-      this.createBubble('leon', event.detail)
+      this.createBubble({
+        who: 'leon',
+        string: event.detail
+      })
+    })
+
+    // Add event delegation for clickable paths
+    this.feed.addEventListener('click', (event) => {
+      if (event.target.classList.contains('clickable-path')) {
+        const path = event.target.getAttribute('data-path')
+        if (path) {
+          this.openPath(path)
+        }
+      }
     })
   }
 
@@ -62,10 +98,7 @@ export default class Chatbot {
   }
 
   loadFeed() {
-    /**
-     * TODO: widget: load widget from local storage
-     */
-    return new Promise((resolve) => {
+    return new Promise(async (resolve) => {
       if (this.parsedBubbles === null || this.parsedBubbles.length === 0) {
         this.noBubbleMessage.classList.remove('hide')
         localStorage.setItem('bubbles', JSON.stringify([]))
@@ -75,7 +108,22 @@ export default class Chatbot {
         for (let i = 0; i < this.parsedBubbles.length; i += 1) {
           const bubble = this.parsedBubbles[i]
 
-          this.createBubble(bubble.who, bubble.string, false)
+          // Skip tool output markers when recreating bubbles
+          if (
+            bubble.originalString &&
+            ToolUIHandler.isToolOutputMarker(bubble.originalString)
+          ) {
+            continue
+          }
+
+          this.createBubble({
+            who: bubble.who,
+            string: bubble.originalString
+              ? bubble.originalString
+              : bubble.string,
+            save: false,
+            isCreatingFromLoadingFeed: true
+          })
 
           if (i + 1 === this.parsedBubbles.length) {
             setTimeout(() => {
@@ -83,43 +131,182 @@ export default class Chatbot {
             }, 100)
           }
         }
+
+        /**
+         * Browse widgets that need to be fetched.
+         * Reverse widgets to fetch the last widgets first.
+         * Replace the loading content with the fetched widget
+         */
+        const widgetContainers = WIDGETS_TO_FETCH.reverse()
+        for (let i = 0; i < widgetContainers.length; i += 1) {
+          const widgetContainer = widgetContainers[i]
+          const hasWidgetBeenFetched = WIDGETS_FETCH_CACHE.has(
+            widgetContainer.widgetId
+          )
+
+          if (hasWidgetBeenFetched) {
+            const fetchedWidget = WIDGETS_FETCH_CACHE.get(
+              widgetContainer.widgetId
+            )
+            widgetContainer.reactRootNode.render(fetchedWidget.reactNode)
+
+            setTimeout(() => {
+              this.scrollDown()
+            }, 100)
+
+            continue
+          }
+
+          const data = await axios.get(
+            `${this.serverURL}/api/v1/fetch-widget?skill_action=${widgetContainer.onFetch.actionName}&widget_id=${widgetContainer.widgetId}`
+          )
+          const fetchedWidget = data.data.widget
+          const reactNode = fetchedWidget
+            ? renderAuroraComponent(
+                this.socket,
+                fetchedWidget.componentTree,
+                fetchedWidget.supportedEvents
+              )
+            : createElement(WidgetWrapper, {
+                children: createElement(Flexbox, {
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  children: createElement(Text, {
+                    secondary: true,
+                    children: 'This widget has been deleted.'
+                  })
+                })
+              })
+
+          widgetContainer.reactRootNode.render(reactNode)
+          WIDGETS_FETCH_CACHE.set(widgetContainer.widgetId, {
+            ...fetchedWidget,
+            reactNode
+          })
+          setTimeout(() => {
+            this.scrollDown()
+          }, 100)
+        }
       }
     })
   }
 
-  createBubble(who, string, save = true) {
+  createBubble(params) {
+    const {
+      who,
+      string,
+      save = true,
+      bubbleId,
+      isCreatingFromLoadingFeed = false,
+      messageId
+    } = params
     const container = document.createElement('div')
     const bubble = document.createElement('p')
 
     container.className = `bubble-container ${who}`
     bubble.className = 'bubble'
-    bubble.innerHTML = string
+
+    if (messageId) {
+      container.setAttribute('data-message-id', messageId)
+    }
+
+    // Store original string before formatting
+    const originalString = string
+    const formattedString = this.formatMessage(string)
+
+    bubble.innerHTML = formattedString
+
+    if (bubbleId) {
+      container.classList.add(bubbleId)
+    }
 
     this.feed.appendChild(container).appendChild(bubble)
 
-    if (container.clientHeight > MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE) {
-      bubble.style.maxHeight = `${MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE}px`
-      const showMore = document.createElement('p')
-      const showMoreText = 'Show more'
+    let widgetComponentTree = null
+    let widgetSupportedEvents = null
 
-      showMore.className = 'show-more'
-      showMore.innerHTML = showMoreText
+    /**
+     * Widget rendering
+     */
+    if (
+      formattedString.includes &&
+      formattedString.includes('"component":"WidgetWrapper"')
+    ) {
+      const parsedWidget = JSON.parse(formattedString)
+      container.setAttribute('data-widget-id', parsedWidget.id)
 
-      container.appendChild(showMore)
+      /**
+       * On widget fetching, render the loader
+       */
+      if (isCreatingFromLoadingFeed && parsedWidget.onFetch) {
+        const root = createRoot(container)
 
-      showMore.addEventListener('click', () => {
-        bubble.classList.toggle('show-all')
-        showMore.innerHTML =
-          showMore.innerHTML === showMoreText ? 'Show less' : showMoreText
-      })
+        root.render(
+          createElement(WidgetWrapper, {
+            children: createElement(Flexbox, {
+              alignItems: 'center',
+              justifyContent: 'center',
+              children: createElement(Loader)
+            })
+          })
+        )
+
+        WIDGETS_TO_FETCH.push({
+          reactRootNode: root,
+          widgetId: parsedWidget.id,
+          onFetch: parsedWidget.onFetch
+        })
+
+        return container
+      }
+
+      widgetComponentTree = parsedWidget.componentTree
+      widgetSupportedEvents = parsedWidget.supportedEvents
+
+      /**
+       * On widget creation
+       */
+      const root = createRoot(container)
+
+      const reactNode = renderAuroraComponent(
+        this.socket,
+        widgetComponentTree,
+        widgetSupportedEvents
+      )
+
+      root.render(reactNode)
     }
 
     if (save) {
-      this.saveBubble(who, string)
+      this.saveBubble(who, originalString, formattedString, messageId)
+    }
+
+    return container
+  }
+
+  handleToolOutput(data) {
+    const result = this.toolUIHandler.handleToolOutput(data)
+
+    // Save to localStorage if it's a new group
+    if (result && result.isNewGroup) {
+      const { toolkitName, toolName, answer } = data
+      const toolInfo = this.toolUIHandler.getToolGroupInfo(
+        result.groupId,
+        toolkitName,
+        toolName,
+        answer
+      )
+
+      this.saveBubble(
+        'leon',
+        toolInfo.originalString,
+        toolInfo.formattedMessage,
+        toolInfo.messageId
+      )
     }
   }
 
-  saveBubble(who, string) {
+  saveBubble(who, originalString, string, messageId) {
     if (!this.noBubbleMessage.classList.contains('hide')) {
       this.noBubbleMessage.classList.add('hide')
     }
@@ -128,8 +315,94 @@ export default class Chatbot {
       this.parsedBubbles.shift()
     }
 
-    this.parsedBubbles.push({ who, string })
+    // Store both original and formatted strings
+    this.parsedBubbles.push({
+      who,
+      string,
+      originalString,
+      messageId
+    })
     localStorage.setItem('bubbles', JSON.stringify(this.parsedBubbles))
     this.scrollDown()
   }
+
+  formatMessage(message) {
+    const isWidget =
+      message.includes && message.includes('"component":"WidgetWrapper"')
+
+    if (typeof message === 'string' && !isWidget) {
+      message = message.replace(/\n/g, '<br />')
+
+      // Handle HTTP/HTTPS URLs with simple regex
+      message = message.replace(/https?:\/\/[^\s<>"{}|\\^`[\]]+/gi, (match) => {
+        return `<a href="${match}" target="_blank" rel="noopener noreferrer" class="clickable-url" title="Open URL in browser">${match}</a>`
+      })
+
+      // Handle file paths with delimiters for exact matching
+      message = message.replace(
+        /\[FILE_PATH\](.*?)\[\/FILE_PATH\]/g,
+        (match, filePath) => {
+          return `<span class="clickable-path" data-path="${filePath}" title="Open in file explorer">${filePath}</span>`
+        }
+      )
+    }
+
+    return message
+  }
+
+  replaceMessage(replaceMessageId, newData) {
+    const existingBubble = document.querySelector(
+      `[data-message-id="${replaceMessageId}"]`
+    )
+
+    if (existingBubble) {
+      existingBubble.remove()
+
+      const bubbleIndex = this.parsedBubbles.findIndex(
+        (bubble) => bubble.messageId === replaceMessageId
+      )
+      if (bubbleIndex !== -1) {
+        this.parsedBubbles.splice(bubbleIndex, 1)
+      }
+    }
+
+    const widgetString =
+      typeof newData === 'string' ? newData : JSON.stringify(newData)
+
+    this.createBubble({
+      who: 'leon',
+      string: widgetString,
+      save: false,
+      messageId: replaceMessageId
+    })
+
+    /**
+     * Only scroll down on the first replacement of this message
+     * to avoid repeating scrolling for every message replacement
+     */
+    if (!REPLACED_MESSAGES.has(replaceMessageId)) {
+      REPLACED_MESSAGES.add(replaceMessageId)
+      this.scrollDown()
+    }
+  }
+
+  openPath(filePath) {
+    // Send request to server to open the file path in system file explorer
+    fetch(`${this.serverURL}/api/v1/open-path`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({ path: filePath })
+    })
+      .then((response) => response.json())
+      .then((data) => {
+        if (!data.success) {
+          console.error('Failed to open path:', data.error)
+        }
+      })
+      .catch((error) => {
+        console.error('Error opening path:', error)
+      })
+  }
 }

app/src/js/client.js

@@ -1,24 +1,29 @@
 import { io } from 'socket.io-client'
-import React from 'react'
-import { createRoot } from 'react-dom/client'
 
-import { Button } from './aurora/button'
 import Chatbot from './chatbot'
+import VoiceEnergy from './voice-energy'
 import { INIT_MESSAGES } from './constants'
+import handleSuggestions from './suggestion-handler.js'
 
 export default class Client {
-  constructor(client, serverUrl, input, res) {
+  constructor(client, serverUrl, input) {
     this.client = client
     this._input = input
     this._suggestionContainer = document.querySelector('#suggestions-container')
+    this.voiceSpeechElement = document.querySelector('#voice-speech')
     this.serverUrl = serverUrl
     this.socket = io(this.serverUrl)
     this.history = localStorage.getItem('history')
     this.parsedHistory = []
-    this.info = res
-    this.chatbot = new Chatbot()
+    this.chatbot = new Chatbot(this.socket, this.serverUrl)
+    this.voiceEnergy = new VoiceEnergy(this)
     this._recorder = {}
     this._suggestions = []
+    this._answerGenerationId = 'xxx'
+    this._ttsAudioContext = null
+    this._isLeonGeneratingAnswer = false
+    this._isVoiceModeEnabled = false
+    // this._ttsAudioContextes = {}
   }
 
   set input(newInput) {
@@ -27,6 +32,10 @@ export default class Client {
     }
   }
 
+  get input() {
+    return this._input
+  }
+
   set recorder(recorder) {
     this._recorder = recorder
   }
@@ -35,6 +44,15 @@ export default class Client {
     return this._recorder
   }
 
+  updateMood(mood) {
+    if (window.leonConfigInfo.llm.enabled) {
+      const moodContainer = document.querySelector('#mood')
+
+      moodContainer.textContent = `Leon's mood: ${mood.emoji}`
+      moodContainer.setAttribute('title', mood.type)
+    }
+  }
+
   async sendInitMessages() {
     for (let i = 0; i < INIT_MESSAGES.length; i++) {
       const messages = INIT_MESSAGES[i]
@@ -53,15 +71,61 @@ export default class Client {
     }
   }
 
-  init(loader) {
+  setInitStatus(statusName, statusType) {
+    window.leonInitStatusEvent.dispatchEvent(
+      new CustomEvent('initStatusChange', {
+        detail: {
+          statusName,
+          statusType
+        }
+      })
+    )
+  }
+
+  asrStartRecording() {
+    if (!window.leonConfigInfo.stt.enabled) {
+      console.warn('ASR is not enabled')
+      return
+    }
+
+    if (!this._isVoiceModeEnabled) {
+      this.enableVoiceMode()
+
+      this.voiceEnergy.status = 'listening'
+
+      this.socket.emit('asr-start-record')
+    }
+  }
+
+  init() {
     this.chatbot.init()
+    this.voiceEnergy.init()
 
     this.socket.on('connect', () => {
       this.socket.emit('init', this.client)
     })
 
+    /**
+     * Init status listeners
+     */
+    this.socket.on('init-client-core-server-handshake', (status) => {
+      this.setInitStatus('clientCoreServerHandshake', status)
+    })
+    this.socket.on('init-tcp-server-boot', (status) => {
+      this.setInitStatus('tcpServerBoot', status)
+    })
+    this.socket.on('init-llm', (status) => {
+      this.setInitStatus('llm', status)
+    })
+    this.socket.on('warmup-llm-duties', (status) => {
+      this.setInitStatus('llmDutiesWarmUp', status)
+    })
+
     this.socket.on('ready', () => {
-      loader.stop()
+      setTimeout(() => {
+        const body = document.querySelector('body')
+        body.classList.remove('settingup')
+      }, 250)
 
       if (this.chatbot.parsedBubbles?.length === 0) {
         this.sendInitMessages()
@@ -69,13 +133,81 @@ export default class Client {
     })
 
     this.socket.on('answer', (data) => {
-      this.chatbot.receivedFrom('leon', data)
+      /*if (this._isVoiceModeEnabled) {
+        this.voiceEnergy.status = 'listening'
+      }*/
+
+      // Leon has finished to answer
+      this._isLeonGeneratingAnswer = false
+
+      /**
+       * Handle message replacement if replaceMessageId is provided
+       */
+      if (data.replaceMessageId) {
+        this.chatbot.replaceMessage(data.replaceMessageId, data)
+        return
+      }
+
+      /**
+       * Handle tool output messages
+       */
+      if (data.isToolOutput) {
+        this.chatbot.handleToolOutput(data)
+        return
+      }
+
+      /**
+       * Handle widget data directly
+       */
+      if (data.widget || data.componentTree) {
+        // Pass the entire widget data as JSON string for chatbot.js to handle
+        const widgetString =
+          typeof data === 'string' ? data : JSON.stringify(data)
+
+        this.chatbot.createBubble({
+          who: 'leon',
+          string: widgetString,
+          messageId: data.widget?.id || data.id || `msg-${Date.now()}`
+        })
+
+        return
+      }
+
+      /**
+       * Just save the bubble if the newest bubble is from the streaming.
+       * Otherwise, create a new bubble
+       */
+      const newestBubbleContainerElement =
+        document.querySelector('.leon:last-child')
+      const isNewestBubbleFromStreaming =
+        newestBubbleContainerElement?.classList.contains(
+          this._answerGenerationId
+        )
+
+      if (isNewestBubbleFromStreaming) {
+        this.chatbot.saveBubble('leon', data)
+
+        // Slightly delay the update to avoid the stream animation to be interrupted
+        setTimeout(() => {
+          // Update the text of the bubble (quick emoji fix)
+          newestBubbleContainerElement.querySelector('p.bubble').innerHTML =
+            this.chatbot.formatMessage(data)
+        }, 2_500)
+      } else {
+        this.chatbot.receivedFrom('leon', data)
+      }
     })
 
     this.socket.on('suggest', (data) => {
-      data?.forEach((suggestionText) => {
+      setTimeout(() => {
+        handleSuggestions(data, this.chatbot, this)
+      }, 400)
+      setTimeout(() => {
+        this.chatbot.scrollDown()
+      }, 450)
+      /*data?.forEach((suggestionText) => {
         this.addSuggestion(suggestionText)
-      })
+      })*/
     })
 
     this.socket.on('is-typing', (data) => {
@@ -89,28 +221,115 @@ export default class Client {
       cb('string-received')
     })
 
-    this.socket.on('widget', (data) => {
-      /**
-       * TODO: widget: widget handler to core/skill; dynamic component rendering
-       */
-      console.log('data', data)
+    this.socket.on('widget-send-utterance', (utterance) => {
+      this._input.value = utterance
+      this.send('utterance')
+    })
 
-      const container = document.createElement('div')
-      container.className = 'widget'
-      this.chatbot.feed.appendChild(container)
+    this.socket.on('new-mood', (mood) => {
+      this.updateMood(mood)
+    })
 
-      const root = createRoot(container)
-
-      // TODO: widget: pass props and dynamic component loading according to type
-      const widgets = {
-        Button: (options) => {
-          return Button({
-            children: options.text
-          })
-        }
+    this.socket.on('llm-token', (data) => {
+      if (this._isVoiceModeEnabled) {
+        this.voiceEnergy.status = 'processing'
       }
 
-      root.render(widgets[data.type](data.options))
+      this._isLeonGeneratingAnswer = true
+      const previousGenerationId = this._answerGenerationId
+      const newGenerationId = data.generationId
+      this._answerGenerationId = newGenerationId
+      const isSameGeneration = previousGenerationId === newGenerationId
+      let bubbleContainerElement = null
+
+      if (!isSameGeneration) {
+        bubbleContainerElement = this.chatbot.createBubble({
+          who: 'leon',
+          string: data.token,
+          save: false,
+          bubbleId: newGenerationId
+        })
+      } else {
+        bubbleContainerElement = document.querySelector(
+          `.${previousGenerationId}`
+        )
+      }
+
+      const bubbleElement = bubbleContainerElement.querySelector('p.bubble')
+
+      // Token is already appened when it's a new generation
+      if (isSameGeneration) {
+        // bubbleElement.textContent += data.token
+
+        const tokenSpan = document.createElement('span')
+        tokenSpan.className = 'llm-token fade-in'
+        tokenSpan.textContent = data.token
+
+        bubbleElement.appendChild(tokenSpan)
+      }
+
+      this.chatbot.scrollDown()
+    })
+
+    this.socket.on('asr-speech', (text) => {
+      if (!this._isVoiceModeEnabled) {
+        this.enableVoiceMode()
+      }
+
+      this.voiceEnergy.status = 'listening'
+      this._input.value = text
+
+      if (this.voiceSpeechElement) {
+        this.voiceSpeechElement.textContent = text
+      }
+    })
+
+    this.socket.on('asr-end-of-owner-speech', () => {
+      this.voiceEnergy.status = 'processing'
+
+      setTimeout(() => {
+        this.send('utterance')
+      }, 200)
+    })
+
+    this.socket.on('asr-active-listening-disabled', () => {
+      this.voiceEnergy.status = 'idle'
+    })
+
+    /**
+     * Only used for "local" TTS provider as a PoC for now.
+     * Target to do a better implementation in the future
+     * with streaming support
+     */
+    this.socket.on('tts-stream', (data) => {
+      this.voiceEnergy.status = 'talking'
+
+      // const { audioId, chunk } = data
+      const { chunk } = data
+      this._ttsAudioContext = new AudioContext()
+      // this._ttsAudioContextes[audioId] = ctx
+
+      const source = this._ttsAudioContext.createBufferSource()
+      this._ttsAudioContext.decodeAudioData(chunk, (buffer) => {
+        source.buffer = buffer
+
+        source.connect(this._ttsAudioContext.destination)
+        source.start(0)
+      })
+    })
+
+    /**
+     * When Leon got interrupted by the owner voice
+     * while he is speaking
+     */
+    this.socket.on('tts-interruption', async () => {
+      if (this._ttsAudioContext) {
+        await this._ttsAudioContext.close()
+      }
+    })
+
+    this.socket.on('tts-end-of-speech', async () => {
+      this.voiceEnergy.status = 'listening'
     })
 
     this.socket.on('audio-forwarded', (data, cb) => {
@@ -127,7 +346,7 @@ export default class Client {
          * When the after speech option is enabled and
          * the answer is a final one
          */
-        if (this.info.after_speech && data.is_final_answer) {
+        if (window.leonConfigInfo.after_speech && data.is_final_answer) {
           // Enable recording after the speech + 500ms
           setTimeout(() => {
             this._recorder.start()
@@ -164,6 +383,11 @@ export default class Client {
   }
 
   send(keyword) {
+    // Prevent from sending utterance if Leon is still generating text (stream)
+    if (keyword === 'utterance' && this._isLeonGeneratingAnswer) {
+      return false
+    }
+
     if (this._input.value !== '') {
       this.socket.emit(keyword, {
         client: this.client,
@@ -205,9 +429,13 @@ export default class Client {
     }
 
     this._input.value = ''
+    setTimeout(() => {
+      // Remove the last character to avoid the space
+      this._input.value = this._input.value.slice(0, -1)
+    }, 0)
   }
 
-  addSuggestion(text) {
+  /*addSuggestion(text) {
     const newSuggestion = document.createElement('button')
     newSuggestion.classList.add('suggestion')
     newSuggestion.textContent = text
@@ -221,5 +449,41 @@ export default class Client {
     })
 
     this._suggestions.push(newSuggestion)
+  }*/
+
+  enableVoiceMode() {
+    if (!this._isVoiceModeEnabled) {
+      this._isVoiceModeEnabled = true
+
+      const body = document.querySelector('body')
+      if (!body.classList.contains('voice-mode-enabled')) {
+        body.classList.add('voice-mode-enabled')
+
+        const voiceOverlayTransitor = document.createElement('div')
+        voiceOverlayTransitor.id = 'voice-overlay-transitor'
+        body.appendChild(voiceOverlayTransitor)
+        voiceOverlayTransitor.addEventListener('animationend', () => {
+          voiceOverlayTransitor.removeEventListener('animationend', () => {})
+          voiceOverlayTransitor.remove()
+        })
+      }
+    }
+  }
+  disableVoiceMode() {
+    if (this._isVoiceModeEnabled) {
+      this._isVoiceModeEnabled = false
+
+      const body = document.querySelector('body')
+
+      const voiceContainer = document.querySelector('#voice-container')
+      if (voiceContainer) {
+        voiceContainer.style.animation = 'none'
+        voiceContainer.style.animation = null
+      }
+
+      if (body.classList.contains('voice-mode-enabled')) {
+        body.classList.remove('voice-mode-enabled')
+      }
+    }
   }
 }

app/src/js/constants.js

@@ -12,9 +12,6 @@ export const INIT_MESSAGES = [
   [
     `Come hang out with us <a href="https://discord.gg/MNQqqKg" target="_blank">on Discord</a>! Once we release our official version, our community will be working together to build new skills for me. You won't want to miss out on the fun!`
   ],
-  [
-    `At the moment, I'm not using a large language model, but we're planning to incorporate some in the future to improve my abilities. You can <a href="https://github.com/leon-ai/leon#how-about-large-language-models-and-leon" target="_blank">learn more about our plans here</a>.`
-  ],
   [
     `Just so you know, my creator is working tirelessly to improve my skills and features, dedicating 75% of his free time to the project on top of his full-time job. If you'd like to help speed up my development, you can sponsor his work by clicking on this link: <strong><a href='http://sponsor.getleon.ai/' target='_blank'>sponsor.getleon.ai</a></strong>. Your support would mean a lot to us. Thank you for choosing me as your assistant!`
   ]

app/src/js/init.jsx

@@ -0,0 +1,193 @@
+import { useEffect, useState, useRef } from 'react'
+import { createRoot } from 'react-dom/client'
+import {
+  WidgetWrapper,
+  Text,
+  Icon,
+  Flexbox,
+  List,
+  ListHeader,
+  ListItem,
+  Loader
+} from '@leon-ai/aurora'
+
+const container = document.querySelector('#init')
+const root = createRoot(container)
+
+function Item({ children, status }) {
+  if (status === 'error') {
+    return <ErrorListItem>{children}</ErrorListItem>
+  }
+  if (status === 'warning') {
+    return <WarningListItem>{children}</WarningListItem>
+  }
+  if (status === 'success') {
+    return <SuccessListItem>{children}</SuccessListItem>
+  }
+  if (status === 'loading') {
+    return <LoadingListItem>{children}</LoadingListItem>
+  }
+
+  return <ListItem>{children}</ListItem>
+}
+
+function LoadingListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Loader size="sm" />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function ErrorListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="close"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="red"
+          bgColor="transparent-red"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function WarningListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="alert"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="yellow"
+          bgColor="transparent-yellow"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function SuccessListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="check"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="green"
+          bgColor="transparent-green"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+
+function Init() {
+  const parentRef = useRef(null)
+  const [config, setConfig] = useState(() => ({ ...window.leonConfigInfo }))
+  const [statusMap, setStatusMap] = useState({
+    clientCoreServerHandshake: 'loading',
+    tcpServerBoot: 'loading',
+    llm: 'loading',
+    llmDutiesWarmUp: 'loading'
+  })
+
+  useEffect(() => {
+    setTimeout(() => {
+      if (parentRef.current) {
+        parentRef.current.classList.remove('not-initialized')
+      }
+    }, 250)
+
+    function handleStatusChange(event) {
+      const { statusName, statusType } = event.detail
+
+      setStatusMap((prev) => ({ ...prev, [statusName]: statusType }))
+    }
+
+    window.leonInitStatusEvent.addEventListener(
+      'initStatusChange',
+      handleStatusChange
+    )
+    return () =>
+      window.leonInitStatusEvent.removeEventListener(
+        'initStatusChange',
+        handleStatusChange
+      )
+  }, [])
+
+  const statuses = []
+  for (let key of Object.keys(statusMap)) {
+    // If LLM is not enabled, we don't need to check for LLM duties warm up
+    if (
+      key === 'llmDutiesWarmUp' &&
+      (!config.llm?.enabled || !config.shouldWarmUpLLMDuties)
+    ) {
+      statuses.push('success')
+    } else if (!config[key] || config[key].enabled) {
+      statuses.push(statusMap[key])
+    }
+  }
+
+  const areAllStatusesSuccess = statuses.every((status) => status === 'success')
+
+  useEffect(() => {
+    if (window.leonConfigInfo) {
+      setConfig({ ...window.leonConfigInfo })
+    }
+  }, [window.leonConfigInfo])
+
+  return (
+    <div
+      style={{
+        position: 'fixed',
+        width: '100vw',
+        height: '100vh',
+        zIndex: 9999,
+        backgroundColor: 'var(--black-color)'
+      }}
+      ref={parentRef}
+      className={areAllStatusesSuccess ? 'initialized' : 'not-initialized'}
+    >
+      <div
+        style={{
+          position: 'absolute',
+          top: '33%',
+          left: '50%',
+          transform: 'translate(-50%, -50%)'
+        }}
+      >
+        <WidgetWrapper noPadding>
+          <List>
+            <ListHeader>Leon is getting ready...</ListHeader>
+            <Item status={statusMap.clientCoreServerHandshake}>
+              Client and core server handshaked
+            </Item>
+            <Item status={statusMap.tcpServerBoot}>TCP server booted</Item>
+            {config.llm && config.llm.enabled && (
+              <Item status={statusMap.llm}>LLM loaded</Item>
+            )}
+            {config.shouldWarmUpLLMDuties && (
+              <Item status={statusMap.llmDutiesWarmUp}>
+                LLM duties warmed up
+              </Item>
+            )}
+          </List>
+        </WidgetWrapper>
+      </div>
+    </div>
+  )
+}
+
+root.render(<Init />)

app/src/js/loader.js

@@ -1,22 +0,0 @@
-export default class Loader {
-  constructor() {
-    this.et = new EventTarget()
-    this.body = document.querySelector('body')
-
-    this.et.addEventListener('settingup', (event) => {
-      if (event.detail) {
-        this.body.classList.add('settingup')
-      } else {
-        this.body.classList.remove('settingup')
-      }
-    })
-  }
-
-  start() {
-    this.et.dispatchEvent(new CustomEvent('settingup', { detail: true }))
-  }
-
-  stop() {
-    this.et.dispatchEvent(new CustomEvent('settingup', { detail: false }))
-  }
-}

app/src/js/main.js

@@ -1,10 +1,13 @@
 import axios from 'axios'
+import '@leon-ai/aurora/style.css'
 
-import Loader from './loader'
+window.leonInitStatusEvent = new EventTarget()
+
+import './init'
 import Client from './client'
-import Recorder from './recorder'
-import listener from './listener'
-import { onkeydowndocument, onkeydowninput } from './onkeydown'
+// import Recorder from './recorder'
+// import listener from './listener'
+import { onkeydownstartrecording, onkeydowninput } from './onkeydown'
 
 const config = {
   app: 'webapp',
@@ -19,29 +22,54 @@ const serverUrl =
     : `${config.server_host}:${config.server_port}`
 
 document.addEventListener('DOMContentLoaded', async () => {
-  const loader = new Loader()
-
-  loader.start()
-
   try {
     const response = await axios.get(`${serverUrl}/api/v1/info`)
     const input = document.querySelector('#utterance')
     const mic = document.querySelector('#mic-button')
     const v = document.querySelector('#version small')
-    const client = new Client(config.app, serverUrl, input, response.data)
-    let rec = {}
-    let chunks = []
+    const infoButton = document.querySelector('#info')
+    const client = new Client(config.app, serverUrl, input)
+    // let rec = {}
+    // let chunks = []
 
-    v.innerHTML += client.info.version
+    window.leonConfigInfo = response.data
+    const infoKeys = [
+      'timeZone',
+      'telemetry',
+      'gpu',
+      'graphicsComputeAPI',
+      'totalVRAM',
+      'freeVRAM',
+      'usedVRAM',
+      'llm',
+      'shouldWarmUpLLMDuties',
+      'isLLMActionRecognitionEnabled',
+      'isLLMNLGEnabled',
+      'stt',
+      'tts',
+      'mood',
+      'version'
+    ]
+    const infoToDisplay = {}
+    infoKeys.forEach((key) => {
+      infoToDisplay[key] = window.leonConfigInfo[key]
+    })
 
-    client.init(loader)
+    v.textContent += window.leonConfigInfo.version
 
-    if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+    client.updateMood(window.leonConfigInfo.mood)
+    client.init()
+
+    infoButton.addEventListener('click', () => {
+      alert(JSON.stringify(infoToDisplay, null, 2))
+    })
+
+    /*if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
       navigator.mediaDevices
         .getUserMedia({ audio: true })
         .then((stream) => {
           if (MediaRecorder) {
-            rec = new Recorder(stream, mic, client.info)
+            rec = new Recorder(stream, mic, window.leonConfigInfo)
             client.recorder = rec
 
             rec.ondataavailable((e) => {
@@ -49,7 +77,7 @@ document.addEventListener('DOMContentLoaded', async () => {
             })
 
             rec.onstart(() => {
-              /* */
+              /!* *!/
             })
 
             rec.onstop(() => {
@@ -106,18 +134,19 @@ document.addEventListener('DOMContentLoaded', async () => {
       console.error(
         'MediaDevices.getUserMedia() is not supported on your browser.'
       )
-    }
+    }*/
 
     document.addEventListener('keydown', (e) => {
-      onkeydowndocument(e, () => {
-        if (rec.enabled === false) {
+      onkeydownstartrecording(e, () => {
+        client.asrStartRecording()
+        /*if (rec.enabled === false) {
           input.value = ''
           rec.start()
           rec.enabled = true
         } else {
           rec.stop()
           rec.enabled = false
-        }
+        }*/
       })
     })
 
@@ -128,13 +157,15 @@ document.addEventListener('DOMContentLoaded', async () => {
     mic.addEventListener('click', (e) => {
       e.preventDefault()
 
-      if (rec.enabled === false) {
+      client.asrStartRecording()
+
+      /*if (rec.enabled === false) {
         rec.start()
         rec.enabled = true
       } else {
         rec.stop()
         rec.enabled = false
-      }
+      }*/
     })
   } catch (e) {
     alert(`Error: ${e.message}; ${JSON.stringify(e.response?.data)}`)

app/src/js/onkeydown.js

@@ -8,29 +8,31 @@ const onkeydowninput = (e, client) => {
     parsedHistory = JSON.parse(localStorage.getItem('history')).reverse()
   }
 
-  if (key === 13) {
+  if (key === 13 && !e.shiftKey) {
     if (client.send('utterance')) {
       parsedHistory = JSON.parse(localStorage.getItem('history')).reverse()
       index = -1
     }
   } else if (localStorage.getItem('history') !== null) {
-    if (key === 38 && index < parsedHistory.length - 1) {
-      index += 1
-      client.input = parsedHistory[index]
-    } else if (key === 40 && index - 1 >= 0) {
-      index -= 1
-      client.input = parsedHistory[index]
-    } else if (key === 40 && index - 1 < 0) {
-      client.input = ''
-      index = -1
+    if (e.shiftKey) {
+      if (key === 38 && index < parsedHistory.length - 1) {
+        index += 1
+        client.input = parsedHistory[index]
+      } else if (key === 40 && index - 1 >= 0) {
+        index -= 1
+        client.input = parsedHistory[index]
+      } else if (key === 40 && index - 1 < 0) {
+        client.input = ''
+        index = -1
+      }
     }
   }
 }
 
-const onkeydowndocument = (e, cb) => {
-  if (e.altKey && e.key === 'c') {
+const onkeydownstartrecording = (e, cb) => {
+  if ((e.metaKey || e.altKey) && e.key === 'c') {
     cb()
   }
 }
 
-export { onkeydowninput, onkeydowndocument }
+export { onkeydowninput, onkeydownstartrecording }

app/src/js/render-aurora-component.js

@@ -0,0 +1,51 @@
+import { createElement } from 'react'
+import * as auroraComponents from '@leon-ai/aurora'
+
+import * as customAuroraComponents from '../custom-aurora-components'
+
+export default function renderAuroraComponent(
+  socket,
+  component,
+  supportedEvents
+) {
+  if (component) {
+    let reactComponent = auroraComponents[component.component]
+    /**
+     * Find custom component if a former component is not found
+     */
+    if (!reactComponent) {
+      reactComponent = customAuroraComponents[component.component]
+    }
+
+    if (!reactComponent) {
+      console.error(`Component ${component} not found`)
+    }
+
+    // Check if the browsed component has a supported event and bind it
+    if (reactComponent) {
+      component.events.forEach((event) => {
+        if (supportedEvents.includes(event.type)) {
+          component.props[event.type] = (data) => {
+            const { method } = event
+
+            socket.emit('widget-event', { method, data })
+          }
+        }
+      })
+    }
+
+    // When children is a component, then wrap it in an array to render properly
+    const isComponent = !!component.props?.children?.component
+    if (isComponent) {
+      component.props.children = [component.props.children]
+    }
+
+    if (component.props?.children && Array.isArray(component.props.children)) {
+      component.props.children = component.props.children.map((child) => {
+        return renderAuroraComponent(socket, child, supportedEvents)
+      })
+    }
+
+    return createElement(reactComponent, component.props)
+  }
+}

app/src/js/suggestion-handler.js

@@ -0,0 +1,42 @@
+import { createElement } from 'react'
+import { createRoot } from 'react-dom/client'
+import { WidgetWrapper, List, ListHeader, ListItem } from '@leon-ai/aurora'
+
+export default function handleSuggestions(data, chatbot, client) {
+  const container = document.createElement('div')
+  container.className = `bubble-container leon`
+
+  chatbot.feed.appendChild(container)
+
+  const root = createRoot(container)
+
+  root.render(
+    createElement(WidgetWrapper, {
+      noPadding: true,
+      children: createElement(List, {
+        children: [
+          createElement(ListHeader, {
+            children: 'Suggestions'
+          }),
+          ...data.map((suggestionText) => {
+            return createElement(ListItem, {
+              children: suggestionText,
+              name: 'suggestion',
+              value: suggestionText,
+              onClick: (suggestion) => {
+                const parent = container.parentNode
+
+                if (parent) {
+                  parent.removeChild(container)
+                }
+
+                client.input.value = suggestion.value
+                client.send('utterance')
+              }
+            })
+          })
+        ]
+      })
+    })
+  )
+}

app/src/js/tool-ui-handler.js

@@ -0,0 +1,216 @@
+/**
+ * Tool UI Handler
+ * Manages the display and interaction of tool output in shell-like containers
+ */
+export default class ToolUIHandler {
+  constructor(feedElement, scrollDownCallback, formatMessageCallback) {
+    this.feed = feedElement
+    this.scrollDown = scrollDownCallback
+    this.formatMessage = formatMessageCallback
+    this.toolGroups = new Map() // Track tool group containers
+  }
+
+  /**
+   * Handle tool output messages with shell-like UI
+   */
+  handleToolOutput(data) {
+    const {
+      toolkitName,
+      toolName,
+      toolGroupId,
+      answer,
+      replaceMessageId,
+      key
+    } = data
+
+    // Check if we need to replace an existing message
+    if (replaceMessageId) {
+      this.replaceToolMessage(replaceMessageId, data)
+      return
+    }
+
+    // Extract answer key from the key (take part after last dot)
+    const answerKey = key ? key.split('.').pop() : 'unknown'
+
+    // Create a fallback group ID if none provided
+    const groupId = toolGroupId || `${toolkitName}_${toolName}_${Date.now()}`
+
+    // Get or create tool group container
+    let toolGroupContainer = this.toolGroups.get(groupId)
+
+    if (!toolGroupContainer) {
+      toolGroupContainer = this.createToolGroupContainer(
+        groupId,
+        toolkitName,
+        toolName,
+        answerKey
+      )
+      this.toolGroups.set(groupId, toolGroupContainer)
+    }
+
+    // Add the tool message to the shell output
+    this.addToolMessage(toolGroupContainer, answer)
+
+    // Auto-scroll to bottom
+    this.scrollDown()
+
+    return {
+      groupId,
+      isNewGroup: toolGroupContainer.isNew
+    }
+  }
+
+  /**
+   * Create a new tool group container
+   */
+  createToolGroupContainer(groupId, toolkitName, toolName, answerKey) {
+    // Create new tool group container
+    const groupContainer = document.createElement('div')
+    groupContainer.className = 'tool-group-container'
+    groupContainer.setAttribute('data-tool-group-id', groupId)
+
+    // Create tool header (expandable)
+    const toolHeader = document.createElement('div')
+    toolHeader.className = 'tool-header'
+    toolHeader.innerHTML = `
+      <i class="ri-terminal-line tool-icon"></i>
+      <span class="tool-name">${toolkitName} toolkit → ${toolName} → ${answerKey}</span>
+      <i class="ri-arrow-down-s-line expand-icon"></i>
+    `
+
+    // Create tool content area
+    const toolContent = document.createElement('div')
+    toolContent.className = 'tool-content'
+
+    // Create shell output area
+    const shellOutput = document.createElement('div')
+    shellOutput.className = 'shell-output'
+
+    toolContent.appendChild(shellOutput)
+    groupContainer.appendChild(toolHeader)
+    groupContainer.appendChild(toolContent)
+
+    // Add expand/collapse functionality
+    this.addExpandCollapseHandler(toolHeader, toolContent)
+
+    // Initially expanded
+    // toolContent.classList.add('expanded')
+    // toolHeader.querySelector('.expand-icon').classList.add('rotated')
+
+    this.feed.appendChild(groupContainer)
+
+    return {
+      container: groupContainer,
+      toolHeader: toolHeader,
+      toolContent: toolContent,
+      shellOutput: shellOutput,
+      isNew: true
+    }
+  }
+
+  /**
+   * Add expand/collapse functionality to tool header
+   */
+  addExpandCollapseHandler(toolHeader, toolContent) {
+    toolHeader.addEventListener('click', () => {
+      const isExpanded = toolContent.classList.contains('expanded')
+      const expandIcon = toolHeader.querySelector('.expand-icon')
+
+      if (isExpanded) {
+        toolContent.classList.remove('expanded')
+        expandIcon.classList.remove('rotated')
+      } else {
+        toolContent.classList.add('expanded')
+        expandIcon.classList.add('rotated')
+      }
+    })
+  }
+
+  /**
+   * Add a tool message to the shell output
+   */
+  addToolMessage(toolGroupContainer, message) {
+    const messageElement = document.createElement('div')
+    messageElement.className = 'shell-message'
+
+    // Format the message
+    const formattedMessage = this.formatMessage(message)
+    messageElement.innerHTML = `<span class="shell-prompt">></span> ${formattedMessage}`
+
+    toolGroupContainer.shellOutput.appendChild(messageElement)
+
+    // Mark as no longer new after first message
+    if (toolGroupContainer.isNew) {
+      toolGroupContainer.isNew = false
+    }
+  }
+
+  /**
+   * Replace a tool message (for progress updates, etc.)
+   */
+  replaceToolMessage(replaceMessageId, newData) {
+    // Find existing tool message by ID
+    const existingMessage = document.querySelector(
+      `[data-message-id="${replaceMessageId}"]`
+    )
+
+    if (existingMessage && existingMessage.closest('.tool-group-container')) {
+      // If it's within a tool container, update just that message
+      const formattedMessage = this.formatMessage(newData.answer)
+      existingMessage.innerHTML = `<span class="shell-prompt">></span> ${formattedMessage}`
+    } else {
+      // Fallback: create new tool output
+      this.handleToolOutput(newData)
+    }
+  }
+
+  /**
+   * Get tool group info for saving to localStorage
+   */
+  getToolGroupInfo(groupId, toolkitName, toolName, message) {
+    return {
+      originalString: `[TOOL_OUTPUT:${groupId}] ${toolkitName} → ${toolName}`,
+      messageId: `tool-${groupId}`,
+      formattedMessage: this.formatMessage(message)
+    }
+  }
+
+  /**
+   * Check if a message is a tool output marker
+   */
+  static isToolOutputMarker(messageString) {
+    return messageString && messageString.startsWith('[TOOL_OUTPUT:')
+  }
+
+  /**
+   * Clear all tool groups (useful for cleanup)
+   */
+  clearToolGroups() {
+    this.toolGroups.clear()
+  }
+
+  /**
+   * Get the number of active tool groups
+   */
+  getToolGroupCount() {
+    return this.toolGroups.size
+  }
+
+  /**
+   * Get a specific tool group by ID
+   */
+  getToolGroup(groupId) {
+    return this.toolGroups.get(groupId)
+  }
+
+  /**
+   * Remove a tool group
+   */
+  removeToolGroup(groupId) {
+    const toolGroup = this.toolGroups.get(groupId)
+    if (toolGroup) {
+      toolGroup.container.remove()
+      this.toolGroups.delete(groupId)
+    }
+  }
+}

app/src/js/voice-energy.js

@@ -0,0 +1,74 @@
+const STATUS = {
+  listening: 'Listening...',
+  processing: 'Processing...',
+  talking: 'Talking...',
+  idle: 'Idle'
+}
+
+export default class VoiceEnergy {
+  constructor(client) {
+    this.client = client
+    this.voiceEnergyContainerElement = document.querySelector(
+      '#voice-energy-container'
+    )
+    this.voiceOverlayElement = document.querySelector('#voice-overlay-bg')
+    this.statusElement = document.querySelector('#voice-status')
+    this._status = 'idle'
+  }
+
+  get status() {
+    return this._status
+  }
+
+  set status(newStatus) {
+    if (this._status !== newStatus) {
+      this._status = newStatus
+
+      if (this.statusElement) {
+        this.statusElement.textContent = STATUS[newStatus]
+      }
+
+      // Clean up speech text when listening
+      if (newStatus === 'listening' && this.client.voiceSpeechElement) {
+        this.client.voiceSpeechElement.textContent = ''
+      }
+
+      if (this.voiceEnergyContainerElement) {
+        this.voiceEnergyContainerElement.className = ''
+        this.voiceEnergyContainerElement.classList.add(newStatus)
+      }
+    }
+  }
+
+  init() {
+    if (this.voiceEnergyContainerElement) {
+      if (this.voiceOverlayElement) {
+        this.voiceOverlayElement.addEventListener('click', (e) => {
+          e.preventDefault()
+          this.client.disableVoiceMode()
+        })
+      }
+
+      const particles = new Set()
+      const particleColors = ['blue', 'pink']
+
+      for (let i = 0; i < 32; i += 1) {
+        const particle = document.createElement('div')
+        const randomColor = Math.floor(Math.random() * 2)
+        let random = Math.floor(Math.random() * 32)
+
+        while (particles.has(random)) {
+          random = Math.floor(Math.random() * 32)
+        }
+
+        particles.add(random)
+        particle.setAttribute('data-particle', String(random))
+        particle.classList.add('voice-particle', particleColors[randomColor])
+        particle.style.transform = `rotate(${
+          i * 11.25
+        }deg) translate(110px) rotate(-${i * 11.25}deg)`
+        this.voiceEnergyContainerElement.appendChild(particle)
+      }
+    }
+  }
+}

app/tsconfig.json

@@ -1,20 +0,0 @@
-{
-  "compilerOptions": {
-    "useDefineForClassFields": true,
-    "skipLibCheck": true,
-    "module": "ESNext",
-    /* Bundler mode */
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "noEmit": true,
-    "jsx": "react-jsx",
-
-    /* Linting */
-    "strict": true,
-    "noUnusedLocals": true,
-    "noUnusedParameters": true,
-    "noFallthroughCasesInSwitch": true
-  }
-}

bin/nvidia/versions.json

@@ -0,0 +1,10 @@
+{
+  "cuda": "12",
+  "cudnn": "9.17.1.4",
+  "cublas": "12.9.1.4",
+  "cusparse": "0.8.1.1",
+  "cusparse_full": "12.5.10.65",
+  "nccl": "2.29.3",
+  "nvshmem": "3.5.19",
+  "nvjitlink": "12.9.86"
+}

bin/pytorch/versions.json

@@ -0,0 +1,3 @@
+{
+  "torch": "2.9.0"
+}

bridges/nodejs/package.json

@@ -2,6 +2,7 @@
   "name": "leon-nodejs-bridge",
   "description": "Leon's Node.js bridge to communicate between the core and skills made with JavaScript",
   "main": "dist/bin/leon-nodejs-bridge.js",
+  "type": "module",
   "author": {
     "name": "Louis Grenard",
     "email": "louis@getleon.ai",
@@ -14,6 +15,7 @@
   },
   "dependencies": {
     "axios": "1.4.0",
+    "ipull": "3.9.2",
     "lodash": "4.17.21"
   },
   "devDependencies": {

bridges/nodejs/src/constants.ts

@@ -1,32 +1,57 @@
 import fs from 'node:fs'
 import path from 'node:path'
 
-import { SKILLS_PATH as SKILLS_ROOT_PATH } from '@/constants'
-import type { SkillConfigSchema } from '@server/schemas/skill-schemas'
+import type { SkillLocaleConfigSchema } from '@/schemas/skill-schemas'
 
-import type { IntentObject } from '@sdk/types'
+import { IntentObject, NLPAction } from '@sdk/types'
 
 const {
   argv: [, , INTENT_OBJ_FILE_PATH]
 } = process
 
+export const LEON_VERSION = process.env['npm_package_version']
+
+const BIN_PATH = path.join(process.cwd(), 'bin')
+const BRIDGES_PATH = path.join(process.cwd(), 'bridges')
+const NODEJS_BRIDGE_ROOT_PATH = path.join(BRIDGES_PATH, 'nodejs')
+const NODEJS_BRIDGE_SRC_PATH = path.join(NODEJS_BRIDGE_ROOT_PATH, 'src')
+const NODEJS_BRIDGE_VERSION_FILE_PATH = path.join(
+  NODEJS_BRIDGE_SRC_PATH,
+  'version.ts'
+)
+
+export const TOOLKITS_PATH = path.join(BRIDGES_PATH, 'toolkits')
+
+export const [, NODEJS_BRIDGE_VERSION] = fs
+  .readFileSync(NODEJS_BRIDGE_VERSION_FILE_PATH, 'utf8')
+  .split("'")
+
 export const INTENT_OBJECT: IntentObject = JSON.parse(
   fs.readFileSync(INTENT_OBJ_FILE_PATH as string, 'utf8')
 )
-export const SKILL_PATH = path.join(
-  SKILLS_ROOT_PATH,
-  INTENT_OBJECT.domain,
-  INTENT_OBJECT.skill
+
+export const NVIDIA_LIBS_PATH = path.join(BIN_PATH, 'nvidia')
+
+export const PYTORCH_PATH = path.join(BIN_PATH, 'pytorch')
+export const PYTORCH_TORCH_PATH = path.join(PYTORCH_PATH, 'torch')
+
+export const SKILLS_PATH = path.join(process.cwd(), 'skills')
+export const SKILL_PATH = path.join(SKILLS_PATH, INTENT_OBJECT.skill_name)
+const SKILL_LOCALE_PATH = path.join(
+  SKILL_PATH,
+  'locales',
+  INTENT_OBJECT.extra_context.lang + '.json'
 )
-export const SKILLS_PATH = SKILLS_ROOT_PATH
-export const SKILL_CONFIG: SkillConfigSchema = JSON.parse(
-  fs.readFileSync(
-    path.join(
-      SKILL_PATH,
-      'config',
-      INTENT_OBJECT.extra_context_data.lang + '.json'
-    ),
-    'utf8'
-  )
+const SKILL_LOCALE_CONFIG_CONTENT = JSON.parse(
+  fs.existsSync(SKILL_LOCALE_PATH)
+    ? fs.readFileSync(SKILL_LOCALE_PATH, 'utf8')
+    : `{"variables": {}, "common_answers": {}, "widget_contents": {}, "actions": {"${INTENT_OBJECT.action_name}": {}}}`
 )
-export { LEON_VERSION, NODEJS_BRIDGE_VERSION } from '@/constants'
+
+export const SKILL_LOCALE_CONFIG: SkillLocaleConfigSchema &
+  SkillLocaleConfigSchema['actions'][NLPAction] = {
+  variables: SKILL_LOCALE_CONFIG_CONTENT.variables,
+  common_answers: SKILL_LOCALE_CONFIG_CONTENT.common_answers,
+  widget_contents: SKILL_LOCALE_CONFIG_CONTENT.widget_contents,
+  ...SKILL_LOCALE_CONFIG_CONTENT.actions[INTENT_OBJECT.action_name]
+}

bridges/nodejs/src/main.ts

@@ -1,49 +1,56 @@
 import path from 'node:path'
 
+import { FileHelper } from '@/helpers/file-helper'
+
 import type { ActionFunction, ActionParams } from '@sdk/types'
 import { INTENT_OBJECT } from '@bridge/constants'
+import { ParamsHelper } from '@sdk/params-helper'
 ;(async (): Promise<void> => {
   const {
-    domain,
-    skill,
-    action,
     lang,
-    utterance,
-    current_entities,
-    entities,
-    current_resolvers,
-    resolvers,
-    slots,
-    extra_context_data
+    sentiment,
+    context_name,
+    skill_name,
+    action_name,
+    skill_config_path,
+    extra_context
   } = INTENT_OBJECT
 
   const params: ActionParams = {
     lang,
-    utterance,
-    current_entities,
-    entities,
-    current_resolvers,
-    resolvers,
-    slots,
-    extra_context_data
+    utterance: INTENT_OBJECT.utterance as ActionParams['utterance'],
+    action_arguments:
+      INTENT_OBJECT.action_arguments as ActionParams['action_arguments'],
+    entities: INTENT_OBJECT.entities as ActionParams['entities'],
+    sentiment,
+    context_name,
+    skill_name,
+    action_name,
+    context: INTENT_OBJECT.context as ActionParams['context'],
+    skill_config: INTENT_OBJECT.skill_config as ActionParams['skill_config'],
+    skill_config_path,
+    extra_context
   }
 
   try {
-    const actionModule = await import(
+    const actionModule = await FileHelper.dynamicImportFromFile(
       path.join(
         process.cwd(),
         'skills',
-        domain,
-        skill,
+        skill_name,
         'src',
         'actions',
-        `${action}.ts`
+        `${action_name}.ts`
       )
     )
     const actionFunction: ActionFunction = actionModule.run
+    const paramsHelper = new ParamsHelper(params)
 
-    await actionFunction(params)
+    await actionFunction(params, paramsHelper)
   } catch (e) {
-    console.error(`Error while running "${skill}" skill "${action}" action:`, e)
+    console.error(
+      `Error while running "${skill_name}" skill "${action_name}" action:`,
+      e
+    )
   }
 })()

bridges/nodejs/src/sdk/aurora/button.ts

@@ -1,12 +1,9 @@
-import { Widget } from '../widget'
+import { type ButtonProps } from '@leon-ai/aurora'
 
-// TODO: contains the button API. rendering engine <-> SDK
-interface ButtonOptions {
-  text: string
-}
+import { WidgetComponent } from '../widget-component'
 
-export class Button extends Widget<ButtonOptions> {
-  public constructor(options: ButtonOptions) {
-    super(options)
+export class Button extends WidgetComponent<ButtonProps> {
+  constructor(props: ButtonProps) {
+    super(props)
   }
 }

bridges/nodejs/src/sdk/aurora/card.ts

@@ -0,0 +1,9 @@
+import { type CardProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Card extends WidgetComponent<CardProps> {
+  constructor(props: CardProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/checkbox.ts

@@ -0,0 +1,9 @@
+import { type CheckboxProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Checkbox extends WidgetComponent<CheckboxProps> {
+  constructor(props: CheckboxProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/circular-progress.ts

@@ -0,0 +1,9 @@
+import { type CircularProgressProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class CircularProgress extends WidgetComponent<CircularProgressProps> {
+  constructor(props: CircularProgressProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/flexbox.ts

@@ -0,0 +1,9 @@
+import { type FlexboxProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Flexbox extends WidgetComponent<FlexboxProps> {
+  constructor(props: FlexboxProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/form.ts

@@ -0,0 +1,9 @@
+import { type FormProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Form extends WidgetComponent<FormProps> {
+  constructor(props: FormProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/icon-button.ts

@@ -0,0 +1,9 @@
+import { type IconButtonProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class IconButton extends WidgetComponent<IconButtonProps> {
+  constructor(props: IconButtonProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/icon.ts

@@ -0,0 +1,9 @@
+import { type IconProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Icon extends WidgetComponent<IconProps> {
+  constructor(props: IconProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/image.ts

@@ -0,0 +1,9 @@
+import { type ImageProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Image extends WidgetComponent<ImageProps> {
+  constructor(props: ImageProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/index.ts

@@ -0,0 +1,30 @@
+export * from './button'
+export * from './card'
+export * from './checkbox'
+export * from './circular-progress'
+export * from './flexbox'
+export * from './form'
+export * from './icon'
+export * from './icon-button'
+export * from './image'
+export * from './input'
+export * from './link'
+export * from './list'
+export * from './list-header'
+export * from './list-item'
+export * from './loader'
+export * from './progress'
+export * from './radio'
+export * from './radio-group'
+export * from './range-slider'
+export * from './scroll-container'
+export * from './select'
+export * from './select-option'
+export * from './status'
+export * from './switch'
+export * from './tab'
+export * from './tab-content'
+export * from './tab-group'
+export * from './tab-list'
+export * from './text'
+export * from './widget-wrapper'

bridges/nodejs/src/sdk/aurora/input.ts

@@ -0,0 +1,9 @@
+import { type InputProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Input extends WidgetComponent<InputProps> {
+  constructor(props: InputProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/link.ts

@@ -0,0 +1,9 @@
+import { type LinkProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Link extends WidgetComponent<LinkProps> {
+  constructor(props: LinkProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list-header.ts

@@ -0,0 +1,9 @@
+import { type ListHeaderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ListHeader extends WidgetComponent<ListHeaderProps> {
+  constructor(props: ListHeaderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list-item.ts

@@ -0,0 +1,9 @@
+import { type ListItemProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ListItem extends WidgetComponent<ListItemProps> {
+  constructor(props: ListItemProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list.ts

@@ -0,0 +1,9 @@
+import { type ListProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class List extends WidgetComponent<ListProps> {
+  constructor(props: ListProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/loader.ts

@@ -0,0 +1,9 @@
+import { type LoaderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Loader extends WidgetComponent<LoaderProps> {
+  constructor(props: LoaderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/progress.ts

@@ -0,0 +1,9 @@
+import { type ProgressProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Progress extends WidgetComponent<ProgressProps> {
+  constructor(props: ProgressProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/radio-group.ts

@@ -0,0 +1,9 @@
+import { type RadioGroupProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class RadioGroup extends WidgetComponent<RadioGroupProps> {
+  constructor(props: RadioGroupProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/radio.ts

@@ -0,0 +1,9 @@
+import { type RadioProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Radio extends WidgetComponent<RadioProps> {
+  constructor(props: RadioProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/range-slider.ts

@@ -0,0 +1,9 @@
+import { type RangeSliderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class RangeSlider extends WidgetComponent<RangeSliderProps> {
+  constructor(props: RangeSliderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/scroll-container.ts

@@ -0,0 +1,9 @@
+import { type ScrollContainerProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ScrollContainer extends WidgetComponent<ScrollContainerProps> {
+  constructor(props: ScrollContainerProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/select-option.ts

@@ -0,0 +1,9 @@
+import { type SelectOptionProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class SelectOption extends WidgetComponent<SelectOptionProps> {
+  constructor(props: SelectOptionProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/select.ts

@@ -0,0 +1,9 @@
+import { type SelectProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Select extends WidgetComponent<SelectProps> {
+  constructor(props: SelectProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/status.ts

@@ -0,0 +1,9 @@
+import { type StatusProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Status extends WidgetComponent<StatusProps> {
+  constructor(props: StatusProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/switch.ts

@@ -0,0 +1,9 @@
+import { type SwitchProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Switch extends WidgetComponent<SwitchProps> {
+  constructor(props: SwitchProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-content.ts

@@ -0,0 +1,9 @@
+import { type TabContentProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabContent extends WidgetComponent<TabContentProps> {
+  constructor(props: TabContentProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-group.ts

@@ -0,0 +1,9 @@
+import { type TabGroupProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabGroup extends WidgetComponent<TabGroupProps> {
+  constructor(props: TabGroupProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-list.ts

@@ -0,0 +1,9 @@
+import { type TabListProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabList extends WidgetComponent<TabListProps> {
+  constructor(props: TabListProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab.ts

@@ -0,0 +1,9 @@
+import { type TabProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Tab extends WidgetComponent<TabProps> {
+  constructor(props: TabProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/text.ts

@@ -0,0 +1,9 @@
+import { type TextProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Text extends WidgetComponent<TextProps> {
+  constructor(props: TextProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/widget-wrapper.ts

@@ -0,0 +1,9 @@
+import { type WidgetWrapperProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class WidgetWrapper extends WidgetComponent<WidgetWrapperProps> {
+  constructor(props: WidgetWrapperProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/leon.ts

@@ -1,13 +1,30 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
 import type {
   AnswerData,
   AnswerInput,
   AnswerOutput,
   AnswerConfig
 } from '@sdk/types'
-import { INTENT_OBJECT, SKILL_CONFIG } from '@bridge/constants'
+import { INTENT_OBJECT, SKILL_LOCALE_CONFIG } from '@bridge/constants'
+import { WidgetWrapper } from '@sdk/aurora'
+import { SUPPORTED_WIDGET_EVENTS } from '@sdk/widget-component'
 
 class Leon {
   private static instance: Leon
+  private static globalAnswers = JSON.parse(
+    fs.readFileSync(
+      path.join(
+        process.cwd(),
+        'core',
+        'data',
+        INTENT_OBJECT.lang,
+        'answers.json'
+      ),
+      'utf8'
+    )
+  ).answers
 
   constructor() {
     if (!Leon.instance) {
@@ -15,6 +32,50 @@ class Leon {
     }
   }
 
+  /**
+   * Injects variables into the answer string
+   * @param answer The answer to inject variables into
+   * @param data The data to apply
+   * @example injectVariables('Hello {{ name }}', { name: 'Leon' }) // 'Hello Leon'
+   */
+  private injectVariables(
+    answer: AnswerConfig,
+    data: AnswerData | null
+  ): AnswerConfig {
+    let finalAnswer = answer
+
+    const applyData = (obj: AnswerData): void => {
+      for (const key in obj) {
+        if (typeof finalAnswer === 'string') {
+          finalAnswer = finalAnswer.replaceAll(`{{ ${key} }}`, String(obj[key]))
+        } else {
+          if (finalAnswer.text) {
+            finalAnswer.text = finalAnswer.text.replaceAll(
+              `{{ ${key} }}`,
+              String(obj[key])
+            )
+          }
+          if (finalAnswer.speech) {
+            finalAnswer.speech = finalAnswer.speech.replaceAll(
+              `{{ ${key} }}`,
+              String(obj[key])
+            )
+          }
+        }
+      }
+    }
+
+    if (data) {
+      applyData(data)
+    }
+
+    if (SKILL_LOCALE_CONFIG.variables) {
+      applyData(SKILL_LOCALE_CONFIG.variables)
+    }
+
+    return finalAnswer
+  }
+
   /**
    * Apply data to the answer
    * @param answerKey The answer key
@@ -26,64 +87,27 @@ class Leon {
     data: AnswerData = null
   ): AnswerConfig {
     try {
-      // In case the answer key is a raw answer
-      if (SKILL_CONFIG.answers == null || !SKILL_CONFIG.answers[answerKey]) {
+      const answers =
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-expect-error
+        SKILL_LOCALE_CONFIG.answers?.[answerKey] ??
+        SKILL_LOCALE_CONFIG.common_answers?.[answerKey] ??
+        Leon.globalAnswers?.[answerKey]
+
+      if (!answers) {
         return answerKey
       }
 
-      const answers = SKILL_CONFIG.answers[answerKey] ?? ''
-      let answer: AnswerConfig
+      const answer = Array.isArray(answers)
+        ? answers[Math.floor(Math.random() * answers.length)] ?? ''
+        : answers
 
-      if (Array.isArray(answers)) {
-        answer = answers[Math.floor(Math.random() * answers.length)] ?? ''
-      } else {
-        answer = answers
-      }
-
-      if (data != null) {
-        for (const key in data) {
-          // In case the answer needs speech and text differentiation
-          if (typeof answer !== 'string' && answer.text) {
-            answer.text = answer.text.replaceAll(`%${key}%`, String(data[key]))
-            answer.speech = answer.speech.replaceAll(
-              `%${key}%`,
-              String(data[key])
-            )
-          } else {
-            answer = (answer as string).replaceAll(
-              `%${key}%`,
-              String(data[key])
-            )
-          }
-        }
-      }
-
-      if (SKILL_CONFIG.variables) {
-        const { variables } = SKILL_CONFIG
-
-        for (const key in variables) {
-          // In case the answer needs speech and text differentiation
-          if (typeof answer !== 'string' && answer.text) {
-            answer.text = answer.text.replaceAll(
-              `%${key}%`,
-              String(variables[key])
-            )
-            answer.speech = answer.speech.replaceAll(
-              `%${key}%`,
-              String(variables[key])
-            )
-          } else {
-            answer = (answer as string).replaceAll(
-              `%${key}%`,
-              String(variables[key])
-            )
-          }
-        }
-      }
-
-      return answer
+      return this.injectVariables(answer, data)
     } catch (e) {
-      console.error('Error while setting answer data:', e)
+      console.error(
+        `Error while setting answer data. Please verify that the answer key "${answerKey}" exists in the locale configuration. Details:`,
+        e
+      )
 
       throw e
     }
@@ -94,9 +118,10 @@ class Leon {
    * @param answerInput The answer input
    * @example answer({ key: 'greet' }) // 'Hello world'
    * @example answer({ key: 'welcome', data: { name: 'Louis' } }) // 'Welcome Louis'
-   * @example answer({ key: 'confirm', core: { restart: true } }) // 'Would you like to retry?'
+   * @example answer({ key: 'confirm', core: { next_action: 'guess_the_number_skill:set_up' } }) // 'Would you like to retry?'
+   * @example answer({ key: 'progress', data: { percentage: 50 }, replaceMessageId: 'progress_msg_123' }) // Replace previous progress message
    */
-  public async answer(answerInput: AnswerInput): Promise<void> {
+  public async answer(answerInput: AnswerInput): Promise<string | null> {
     try {
       const answerObject: AnswerOutput = {
         ...INTENT_OBJECT,
@@ -109,20 +134,40 @@ class Leon {
             answerInput.key != null
               ? this.setAnswerData(answerInput.key, answerInput.data)
               : '',
-          core: answerInput.core
+          core: answerInput.core,
+          replaceMessageId: answerInput.replaceMessageId || null
         }
       }
 
       if (answerInput.widget) {
-        answerObject.output.widget = answerInput.widget
+        answerObject.output.widget = {
+          actionName: `${INTENT_OBJECT.skill_name}:${INTENT_OBJECT.action_name}`,
+          widget: answerInput.widget.widget,
+          id: answerInput.widget.id,
+          onFetch: answerInput.widget.onFetch ?? null,
+          componentTree: new WidgetWrapper({
+            ...answerInput.widget.wrapperProps,
+            children: [answerInput.widget.render()]
+          }),
+          supportedEvents: SUPPORTED_WIDGET_EVENTS
+        }
       }
 
       // "Temporize" for the data buffer output on the core
       await new Promise((r) => setTimeout(r, 100))
 
-      process.stdout.write(JSON.stringify(answerObject))
+      // Write the answer object to stdout as a JSON string with a newline for brain chunk-by-chunk parsing
+      process.stdout.write(JSON.stringify(answerObject) + '\n')
+
+      // Return the message ID for future replacement
+      return (
+        answerInput.widget?.id ||
+        `msg-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`
+      )
     } catch (e) {
       console.error('Error while creating answer:', e)
+
+      return null
     }
   }
 }

bridges/nodejs/src/sdk/memory.ts

@@ -24,6 +24,7 @@ export class Memory<T = unknown> {
 
     if (this.name.includes(':') && this.name.split(':').length === 3) {
       this.isFromAnotherSkill = true
+
       const [domainName, skillName, memoryName] = this.name.split(':')
       this.memoryPath = path.join(
         SKILLS_PATH,

bridges/nodejs/src/sdk/network.ts

@@ -18,10 +18,25 @@ interface NetworkRequestOptions {
   method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'
 
   /** Data to be sent as the request body. */
-  data?: Record<string, unknown>
+  data?: unknown
 
   /** Custom headers to be sent. */
   headers?: Record<string, string>
+
+  /** Optional files for multipart/form-data requests (parity with Python SDK). */
+  files?: Record<string, unknown>
+
+  /** Whether to send JSON body (true by default). If false, send form data. */
+  useJson?: boolean
+
+  /** Response type (defaults to 'json'). Use 'arraybuffer' for binary data like audio/video files. */
+  responseType?:
+    | 'json'
+    | 'text'
+    | 'arraybuffer'
+    | 'blob'
+    | 'document'
+    | 'stream'
 }
 
 interface NetworkResponse<ResponseData> {
@@ -35,11 +50,23 @@ interface NetworkResponse<ResponseData> {
   options: NetworkRequestOptions & NetworkOptions
 }
 
+const formatErrorData = (data: unknown): string => {
+  if (typeof data === 'string') {
+    return data
+  }
+
+  try {
+    return JSON.stringify(data)
+  } catch {
+    return String(data)
+  }
+}
+
 export class NetworkError<ResponseErrorData = unknown> extends Error {
   public readonly response: NetworkResponse<ResponseErrorData>
 
-  public constructor(response: NetworkResponse<ResponseErrorData>) {
-    super(`[NetworkError]: ${response.statusCode} ${response.data}`)
+  constructor(response: NetworkResponse<ResponseErrorData>) {
+    super(`[NetworkError]: ${response.statusCode}`)
     this.response = response
     Object.setPrototypeOf(this, NetworkError.prototype)
   }
@@ -49,7 +76,7 @@ export class Network {
   private readonly options: NetworkOptions
   private axios: AxiosInstance
 
-  public constructor(options: NetworkOptions = {}) {
+  constructor(options: NetworkOptions = {}) {
     this.options = options
     this.axios = axios.create({
       baseURL: this.options.baseURL
@@ -65,13 +92,12 @@ export class Network {
     options: NetworkRequestOptions
   ): Promise<NetworkResponse<ResponseData>> {
     try {
-      const response = await this.axios.request<string>({
+      const response = await this.axios.request({
         url: options.url,
         method: options.method.toLowerCase(),
-        data: options.data,
-        transformResponse: (data) => {
-          return data
-        },
+        // For parity, we accept any data type here (including FormData)
+        data: options.data as never,
+        responseType: options.responseType,
         headers: {
           'User-Agent': `Leon Personal Assistant ${LEON_VERSION} - Node.js Bridge ${NODEJS_BRIDGE_VERSION}`,
           ...options.headers
@@ -79,10 +105,24 @@ export class Network {
       })
 
       let data = {} as ResponseData
-      try {
-        data = JSON.parse(response.data)
-      } catch {
+      // For binary response types, return data as-is
+      if (
+        options.responseType === 'arraybuffer' ||
+        options.responseType === 'blob' ||
+        options.responseType === 'stream'
+      ) {
         data = response.data as ResponseData
+      } else {
+        // For text/json responses, try to parse as JSON
+        try {
+          if (typeof response.data === 'string') {
+            data = JSON.parse(response.data)
+          } else {
+            data = response.data as ResponseData
+          }
+        } catch {
+          data = response.data as ResponseData
+        }
       }
 
       return {
@@ -109,14 +149,24 @@ export class Network {
         data = dataRawText as ResponseErrorData
       }
 
-      throw new NetworkError<ResponseErrorData>({
+      const response: NetworkResponse<ResponseErrorData> = {
         data,
         statusCode,
         options: {
           ...this.options,
           ...options
         }
-      })
+      }
+
+      console.error(
+        '[NetworkError]',
+        response.statusCode,
+        options.method,
+        options.url,
+        formatErrorData(response.data)
+      )
+
+      throw new NetworkError<ResponseErrorData>(response)
     }
   }
 

bridges/nodejs/src/sdk/params-helper.ts

@@ -0,0 +1,122 @@
+import type { ActionParams, NEREntity } from '@sdk/types'
+import { INTENT_OBJECT } from '@bridge/constants'
+
+export class ParamsHelper {
+  private readonly params: ActionParams
+
+  constructor(params: ActionParams) {
+    this.params = params
+  }
+
+  /**
+   * Get the widget id if any
+   * @example getWidgetId() // 'timerwidget-5q1xlzeh
+   */
+  getWidgetId(): string | null {
+    return (
+      INTENT_OBJECT.entities?.find((entity) => entity.entity === 'widgetid')
+        ?.sourceText ?? null
+    )
+  }
+
+  /**
+   * Get a specific action argument from the current turn by its name
+   * @param name The name of the action argument to retrieve
+   */
+  getActionArgument(name: string): string | undefined {
+    return this.params.action_arguments[name] as string | undefined
+  }
+
+  /**
+   * Find the first entity in the current turn that matches the given name
+   * @param entityName The name of the entity to find (e.g., 'language', 'date')
+   */
+  findEntity(entityName: string): NEREntity | undefined {
+    return this.params.entities.find((entity) => entity.entity === entityName)
+  }
+
+  /**
+   * Find the last entity in the current turn that matches the given name
+   * Useful when an utterance contains duplicates
+   * @param entityName The name of the entity to find (e.g., 'color')
+   */
+  findLastEntity(entityName: string): NEREntity | undefined {
+    return [...this.params.entities]
+      .reverse()
+      .find((entity) => entity.entity === entityName)
+  }
+
+  /**
+   * Find all entities in the current turn that match the given name
+   * @param entityName The name of the entities to find (e.g., 'date')
+   */
+  findAllEntities(entityName: string): NEREntity[] {
+    return this.params.entities.filter((entity) => entity.entity === entityName)
+  }
+
+  /**
+   * Find the first action argument in the conversation context that matches the given name
+   * @param name The name of the action argument to find
+   */
+  findActionArgumentFromContext(name: string): string | undefined {
+    for (const args of this.params.context.action_arguments) {
+      if (args && name in args) {
+        return args[name] as string | undefined
+      }
+    }
+
+    return undefined
+  }
+
+  /**
+   * Find the most recent value for a given action argument from the conversation context.
+   * It searches backwards from the most recent turn
+   * @param name The name of the action argument to find
+   */
+  findLastActionArgumentFromContext(name: string): string | undefined {
+    // Iterate backwards through the history of action arguments
+    for (
+      let i = this.params.context.action_arguments.length - 1;
+      i >= 0;
+      i -= 1
+    ) {
+      const args = this.params.context.action_arguments[i]
+
+      if (args && name in args) {
+        return args[name] as string | undefined
+      }
+    }
+
+    return undefined
+  }
+
+  /**
+   * Find the most recently detected entity (the last one from the context) that matches the given name.
+   * This is useful for recalling the last time an owner mentioned a specific piece of information
+   * @param entityName The name of the entity to find in the conversation history
+   */
+  findLastEntityFromContext(entityName: string): NEREntity | undefined {
+    // The context.entities are stored chronologically, so reversing and finding the first is correct
+    return [...this.params.context.entities]
+      .reverse()
+      .find((entity) => entity.entity === entityName)
+  }
+
+  /**
+   * Find all historical entities that match the given name from the entire conversation context
+   * @param entityName The name of the entities to find in the conversation history
+   */
+  findAllEntitiesFromContext(entityName: string): NEREntity[] {
+    return this.params.context.entities.filter(
+      (entity) => entity.entity === entityName
+    )
+  }
+
+  /**
+   * Get a value stored in the generic context data store
+   * @param key The key to retrieve
+   */
+  getContextData<T = unknown>(key: string): T | undefined {
+    return this.params.context.data?.[key] as T | undefined
+  }
+}

bridges/nodejs/src/sdk/settings.ts

@@ -7,7 +7,7 @@ export class Settings<T extends Record<string, unknown>> {
   private readonly settingsPath: string
   private readonly settingsSamplePath: string
 
-  public constructor() {
+  constructor() {
     this.settingsPath = path.join(SKILL_PATH, 'src', 'settings.json')
     this.settingsSamplePath = path.join(
       SKILL_PATH,

bridges/nodejs/src/sdk/toolkit-config.ts

@@ -0,0 +1,108 @@
+import { readFileSync, existsSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { getPlatformName } from '@sdk/utils'
+
+interface ToolConfig {
+  description: string
+  binaries?: Record<string, string>
+  resources?: Record<string, string[]>
+}
+
+interface ToolkitConfigData {
+  name: string
+  description: string
+  tools: Record<string, ToolConfig>
+}
+
+export class ToolkitConfig {
+  private static configCache = new Map<string, ToolkitConfigData>()
+  private static settingsCache = new Map<string, Record<string, unknown>>()
+
+  /**
+   * Load tool configuration from bridges/toolkits directory
+   * @param toolkitName - The toolkit name (e.g., 'video_streaming')
+   * @param toolName - Name of the tool (e.g., 'ffmpeg')
+   */
+  static load(toolkitName: string, toolName: string): ToolConfig {
+    const cacheKey = toolkitName
+
+    // Load toolkit config if not cached
+    if (!this.configCache.has(cacheKey)) {
+      const configPath = join(
+        process.cwd(),
+        'bridges',
+        'toolkits',
+        toolkitName,
+        'toolkit.json'
+      )
+      const configContent = readFileSync(configPath, 'utf-8')
+      const config = JSON.parse(configContent) as ToolkitConfigData
+
+      this.configCache.set(cacheKey, config)
+    }
+
+    const toolkitConfig = this.configCache.get(cacheKey)!
+    const toolConfig = toolkitConfig.tools[toolName]
+
+    if (!toolConfig) {
+      throw new Error(
+        `Tool '${toolName}' not found in toolkit '${toolkitConfig.name}'`
+      )
+    }
+
+    return toolConfig
+  }
+
+  /**
+   * Load toolkit settings from bridges/toolkits directory
+   * @param toolkitName - The toolkit name (e.g., 'video_streaming')
+   */
+  static loadSettings(toolkitName: string): Record<string, unknown> {
+    const cacheKey = toolkitName
+
+    if (!this.settingsCache.has(cacheKey)) {
+      const settingsPath = join(
+        process.cwd(),
+        'bridges',
+        'toolkits',
+        toolkitName,
+        'settings.json'
+      )
+
+      let settingsConfig: Record<string, unknown> = {}
+      if (existsSync(settingsPath)) {
+        const settingsContent = readFileSync(settingsPath, 'utf-8')
+        settingsConfig = JSON.parse(settingsContent) as Record<string, unknown>
+      }
+
+      this.settingsCache.set(cacheKey, settingsConfig)
+    }
+
+    return this.settingsCache.get(cacheKey) || {}
+  }
+
+  /**
+   * Load tool-specific settings from toolkit settings file
+   * @param toolkitName - The toolkit name (e.g., 'video_streaming')
+   * @param toolName - Name of the tool (e.g., 'ffmpeg')
+   */
+  static loadToolSettings(
+    toolkitName: string,
+    toolName: string
+  ): Record<string, unknown> {
+    const settings = this.loadSettings(toolkitName)
+    const toolSettings = settings[toolName] as Record<string, unknown>
+
+    return toolSettings && typeof toolSettings === 'object' ? toolSettings : {}
+  }
+
+  /**
+   * Get binary download URL for current platform with architecture granularity
+   */
+  static getBinaryUrl(config: ToolConfig): string | undefined {
+    const platformName = getPlatformName()
+
+    return config.binaries?.[platformName]
+  }
+}

bridges/nodejs/src/sdk/tools/assemblyai_audio/assemblyai_audio-tool.ts

@@ -0,0 +1,268 @@
+import fs from 'node:fs'
+
+import type { TranscriptionOutput } from '@sdk/tools/transcription-schema'
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+import { Network } from '@sdk/network'
+
+// Hardcoded default setting for AssemblyAI audio tool
+// This can be overridden by toolkit settings.json per toolkit.
+const ASSEMBLYAI_AUDIO_API_KEY: string | null = null
+
+interface AssemblyAIUploadResponse {
+  upload_url: string
+}
+
+interface AssemblyAITranscriptionResponse {
+  id: string
+  status: 'queued' | 'processing' | 'completed' | 'error'
+  text: string
+  words?: {
+    text: string
+    start: number
+    end: number
+    confidence: number
+    speaker?: string
+  }[]
+  utterances?: {
+    text: string
+    start: number
+    end: number
+    confidence: number
+    speaker: string
+    words: {
+      text: string
+      start: number
+      end: number
+      confidence: number
+    }[]
+  }[]
+  audio_duration?: number
+  error?: string
+}
+
+export default class AssemblyAIAudioTool extends Tool {
+  private static readonly TOOLKIT = 'music_audio'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+  readonly apiKey: string | null
+
+  constructor() {
+    super()
+    this.config = ToolkitConfig.load(AssemblyAIAudioTool.TOOLKIT, this.toolName)
+
+    const toolSettings = ToolkitConfig.loadToolSettings(
+      AssemblyAIAudioTool.TOOLKIT,
+      this.toolName
+    )
+
+    // Priority: toolkit settings > hardcoded default
+    this.apiKey =
+      (toolSettings['ASSEMBLYAI_AUDIO_API_KEY'] as string) ||
+      ASSEMBLYAI_AUDIO_API_KEY
+  }
+
+  get toolName(): string {
+    return 'assemblyai_audio'
+  }
+
+  get toolkit(): string {
+    return AssemblyAIAudioTool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Transcribe audio to a file using AssemblyAI's audio transcription API via SDK Network
+   * @param inputPath Path to the audio file to transcribe
+   * @param outputPath Path to save the JSON transcription
+   * @param apiKey AssemblyAI API key (uses env/hardcoded default if not provided)
+   * @param speakerLabels Enable speaker diarization (default: true)
+   */
+  async transcribeToFile(
+    inputPath: string,
+    outputPath: string,
+    apiKey?: string,
+    speakerLabels = true
+  ): Promise<string> {
+    // Use provided apiKey, instance apiKey, or error
+    const finalApiKey = apiKey || this.apiKey
+    if (!finalApiKey) {
+      throw new Error('AssemblyAI API key is missing')
+    }
+
+    const network = new Network({ baseURL: 'https://api.assemblyai.com' })
+
+    // Step 1: Upload the audio file
+    const audioData = await fs.promises.readFile(inputPath)
+    const uploadResponse = await network.request({
+      url: '/v2/upload',
+      method: 'POST',
+      data: audioData,
+      headers: {
+        Authorization: finalApiKey,
+        'Content-Type': 'application/octet-stream'
+      }
+    })
+
+    const uploadUrl = (uploadResponse.data as AssemblyAIUploadResponse)
+      .upload_url
+
+    // Step 2: Submit transcription request
+    const transcriptionResponse = await network.request({
+      url: '/v2/transcript',
+      method: 'POST',
+      data: {
+        audio_url: uploadUrl,
+        speaker_labels: speakerLabels
+      },
+      headers: {
+        Authorization: finalApiKey,
+        'Content-Type': 'application/json'
+      }
+    })
+
+    const transcriptId = (
+      transcriptionResponse.data as AssemblyAITranscriptionResponse
+    ).id
+
+    // Step 3: Poll for completion
+    let transcriptData: AssemblyAITranscriptionResponse
+    let attempts = 0
+    const maxAttempts = 180 // 15 minutes with 5 second intervals
+
+    while (attempts < maxAttempts) {
+      const statusResponse = await network.request({
+        url: `/v2/transcript/${transcriptId}`,
+        method: 'GET',
+        headers: {
+          Authorization: finalApiKey
+        }
+      })
+
+      transcriptData = statusResponse.data as AssemblyAITranscriptionResponse
+
+      if (transcriptData.status === 'completed') {
+        break
+      } else if (transcriptData.status === 'error') {
+        throw new Error(
+          `AssemblyAI transcription failed: ${
+            transcriptData.error || 'Unknown error'
+          }`
+        )
+      }
+
+      // Wait 5 seconds before polling again
+      await new Promise((resolve) => setTimeout(resolve, 5000))
+      attempts++
+    }
+
+    if (attempts >= maxAttempts) {
+      throw new Error('AssemblyAI transcription timed out')
+    }
+
+    // Step 4: Parse and save the transcription
+    const parsedOutput = this.parseTranscription(transcriptData!)
+
+    await fs.promises.writeFile(
+      outputPath,
+      JSON.stringify(parsedOutput, null, 2),
+      'utf8'
+    )
+
+    return outputPath
+  }
+
+  private parseTranscription(
+    rawOutput: AssemblyAITranscriptionResponse
+  ): TranscriptionOutput {
+    const segments: {
+      from: number
+      to: number
+      text: string
+      speaker: string | null
+    }[] = []
+    const speakers: Set<string> = new Set()
+
+    // Use utterances for speaker-labeled segments if available
+    if (rawOutput.utterances && rawOutput.utterances.length > 0) {
+      for (const utterance of rawOutput.utterances) {
+        segments.push({
+          from: utterance.start / 1_000, // Convert milliseconds to seconds
+          to: utterance.end / 1_000,
+          text: utterance.text,
+          speaker: utterance.speaker
+        })
+        speakers.add(utterance.speaker)
+      }
+    } else if (rawOutput.words && rawOutput.words.length > 0) {
+      // Fallback to word-level data if utterances are not available
+      // Group consecutive words by speaker (if available)
+      let currentSegment: {
+        from: number
+        to: number
+        text: string
+        speaker: string | null
+      } | null = null
+
+      for (const word of rawOutput.words) {
+        const speaker = word.speaker || null
+
+        if (
+          currentSegment &&
+          currentSegment.speaker === speaker &&
+          word.start / 1_000 - currentSegment.to < 1.0 // Max 1 second gap
+        ) {
+          // Extend current segment
+          currentSegment.to = word.end / 1_000
+          currentSegment.text += ` ${word.text}`
+        } else {
+          // Start a new segment
+          if (currentSegment) {
+            segments.push(currentSegment)
+          }
+          currentSegment = {
+            from: word.start / 1_000,
+            to: word.end / 1_000,
+            text: word.text,
+            speaker: speaker
+          }
+        }
+
+        if (speaker) {
+          speakers.add(speaker)
+        }
+      }
+
+      // Push the last segment
+      if (currentSegment) {
+        segments.push(currentSegment)
+      }
+    } else {
+      // Fallback: create a single segment with the full text
+      segments.push({
+        from: 0,
+        to: (rawOutput.audio_duration || 0) / 1_000,
+        text: rawOutput.text,
+        speaker: null
+      })
+    }
+
+    // Calculate duration
+    let duration = rawOutput.audio_duration ? rawOutput.audio_duration : 0
+    if (!duration && segments.length > 0) {
+      duration = segments[segments.length - 1]?.to || 0
+    }
+
+    return {
+      duration,
+      speakers: Array.from(speakers),
+      speaker_count: speakers.size,
+      segments,
+      metadata: {
+        tool: this.toolName
+      }
+    }
+  }
+}

bridges/nodejs/src/sdk/tools/assemblyai_audio/index.ts

@@ -0,0 +1 @@
+export { default } from './assemblyai_audio-tool'

bridges/nodejs/src/sdk/tools/bash/bash-tool.ts

@@ -0,0 +1,267 @@
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+
+interface BashResult {
+  success: boolean
+  stdout: string
+  stderr: string
+  returncode: number
+  command: string
+}
+
+interface ExecuteOptions {
+  cwd?: string
+  timeout?: number
+  captureOutput?: boolean
+}
+
+export default class BashTool extends Tool {
+  private static readonly TOOLKIT = 'operating_system_control'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+
+  constructor() {
+    super()
+    // Load configuration from central toolkits directory
+    const toolConfigName = this.constructor.name
+      .toLowerCase()
+      .replace('tool', '')
+    this.config = ToolkitConfig.load(BashTool.TOOLKIT, toolConfigName)
+  }
+
+  get toolName(): string {
+    return this.constructor.name
+  }
+
+  get toolkit(): string {
+    return BashTool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Execute a bash command and return the result.
+   */
+  async executeBashCommand(
+    command: string,
+    options: ExecuteOptions = {}
+  ): Promise<BashResult> {
+    const { cwd = process.cwd(), timeout = 30 } = options
+
+    try {
+      // Use the base tool's command execution method
+      // For bash commands, we'll use 'bash' as the binary and '-c' with the command as args
+      const resultOutput = await this.executeCommand({
+        binaryName: 'bash',
+        args: ['-c', command],
+        options: {
+          sync: true,
+          cwd,
+          timeout: timeout * 1_000
+        },
+        skipBinaryDownload: true // bash is a built-in command, no need to download
+      })
+
+      return {
+        success: true,
+        stdout: resultOutput.trim(),
+        stderr: '',
+        returncode: 0,
+        command
+      }
+    } catch (error: unknown) {
+      const errorMessage = (error as Error).message
+
+      // Parse error to determine if it was a timeout, command failure, or other error
+      if (errorMessage.toLowerCase().includes('timed out')) {
+        return {
+          success: false,
+          stdout: '',
+          stderr: `Command timed out after ${timeout} seconds`,
+          returncode: -1,
+          command
+        }
+      } else if (errorMessage.includes('failed with exit code')) {
+        // Extract exit code and error from the base tool's error message
+        const exitCodeMatch = errorMessage.match(/exit code (\d+)/)
+        const exitCode =
+          exitCodeMatch && exitCodeMatch[1]
+            ? parseInt(exitCodeMatch[1], 10)
+            : -1
+
+        // Extract stderr from the error message if present
+        const stderrMatch = errorMessage.match(/exit code \d+: (.+)$/)
+        const stderr =
+          stderrMatch && stderrMatch[1] ? stderrMatch[1] : errorMessage
+
+        return {
+          success: false,
+          stdout: '',
+          stderr,
+          returncode: exitCode,
+          command
+        }
+      } else {
+        return {
+          success: false,
+          stdout: '',
+          stderr: errorMessage,
+          returncode: -1,
+          command
+        }
+      }
+    }
+  }
+
+  /**
+   * Basic safety check for bash commands.
+   * Returns True if command appears safe to execute.
+   */
+  async isSafeCommand(command: string): Promise<boolean> {
+    // List of dangerous command patterns
+    const dangerousPatterns = [
+      'rm -rf /',
+      'rm -rf /*',
+      'mkfs',
+      'dd if=',
+      'format',
+      'fdisk',
+      '> /dev/',
+      'chmod 777 /',
+      'chown -R',
+      'kill -9 -1',
+      'killall -9',
+      'fork()',
+      'while true; do',
+      'curl | sh',
+      'wget | sh',
+      '| bash',
+      '| sh',
+      'eval $(curl',
+      'eval $(wget'
+    ]
+
+    const commandLower = command.toLowerCase()
+
+    // Check for dangerous patterns
+    for (const pattern of dangerousPatterns) {
+      if (commandLower.includes(pattern)) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Assess the risk level of a command.
+   * Returns: 'low', 'medium', 'high', or 'critical'
+   */
+  async getCommandRiskLevel(command: string): Promise<string> {
+    const commandLower = command.toLowerCase()
+
+    // Critical risk commands
+    const criticalPatterns = [
+      'rm -rf /',
+      'rm -rf /*',
+      'mkfs',
+      'format',
+      'fdisk',
+      'kill -9 -1'
+    ]
+
+    // High risk commands
+    const highRiskPatterns = [
+      'rm -rf',
+      'rm -f',
+      'chmod 777',
+      'chown -R',
+      'dd if=',
+      'killall',
+      'pkill',
+      'sudo su',
+      'curl | sh',
+      'wget | sh'
+    ]
+
+    // Medium risk commands
+    const mediumRiskPatterns = [
+      'sudo',
+      'rm ',
+      'mv ',
+      'cp ',
+      'chmod',
+      'chown',
+      'install',
+      'apt ',
+      'yum ',
+      'brew ',
+      'pip install'
+    ]
+
+    let riskLevel = 'low'
+
+    for (const pattern of criticalPatterns) {
+      if (commandLower.includes(pattern)) {
+        riskLevel = 'critical'
+        break
+      }
+    }
+
+    if (riskLevel === 'low') {
+      for (const pattern of highRiskPatterns) {
+        if (commandLower.includes(pattern)) {
+          riskLevel = 'high'
+          break
+        }
+      }
+    }
+
+    if (riskLevel === 'low') {
+      for (const pattern of mediumRiskPatterns) {
+        if (commandLower.includes(pattern)) {
+          riskLevel = 'medium'
+          break
+        }
+      }
+    }
+
+    return riskLevel
+  }
+
+  /**
+   * Get a human-readable description of the command's risk.
+   */
+  async getRiskDescription(command: string): Promise<string> {
+    const riskLevel = await this.getCommandRiskLevel(command)
+    const commandLower = command.toLowerCase()
+
+    if (commandLower.includes('rm')) {
+      return 'delete files or directories permanently'
+    } else if (commandLower.includes('sudo')) {
+      return 'make system-level changes with elevated privileges'
+    } else if (commandLower.includes('kill')) {
+      return 'terminate running processes'
+    } else if (
+      commandLower.includes('chmod') ||
+      commandLower.includes('chown')
+    ) {
+      return 'change file permissions or ownership'
+    } else if (
+      ['apt', 'yum', 'brew', 'pip'].some((pkg) => commandLower.includes(pkg))
+    ) {
+      return 'install or modify system packages'
+    } else if (commandLower.includes('curl') || commandLower.includes('wget')) {
+      return 'download content from the internet'
+    } else {
+      const descriptions: Record<string, string> = {
+        critical: 'cause severe system damage',
+        high: 'cause significant system changes',
+        medium: 'modify your system',
+        low: 'perform system operations'
+      }
+      return descriptions[riskLevel] || 'affect your system'
+    }
+  }
+}

bridges/nodejs/src/sdk/tools/bash/index.ts

@@ -0,0 +1 @@
+export { default } from './bash-tool'

bridges/nodejs/src/sdk/tools/cerebras/cerebras-tool.ts

@@ -0,0 +1,345 @@
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+import { Network, NetworkError } from '@sdk/network'
+
+// Hardcoded default settings for Cerebras tool
+// These can be overridden by toolkit settings.json per toolkit.
+const CEREBRAS_API_KEY: string | null = null
+const CEREBRAS_MODEL = 'zai-glm-4.7'
+
+interface ChatMessage {
+  role: string
+  content: string
+}
+
+interface ChatCompletionOptions {
+  messages: ChatMessage[]
+  model?: string
+  temperature?: number
+  max_tokens?: number
+  system_prompt?: string
+  use_structured_output?: boolean
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  json_schema?: Record<string, any>
+}
+
+interface CompletionOptions {
+  prompt: string
+  model?: string
+  temperature?: number
+  max_tokens?: number
+  system_prompt?: string
+  use_structured_output?: boolean
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  json_schema?: Record<string, any>
+}
+
+interface StructuredCompletionOptions {
+  prompt: string
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  json_schema: Record<string, any>
+  model?: string
+  temperature?: number
+  max_tokens?: number
+  system_prompt?: string
+}
+
+interface ApiResponse {
+  success: boolean
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  data?: any
+  model_used?: string
+  error?: string
+  status_code?: number
+}
+
+export default class CerebrasTool extends Tool {
+  private static readonly TOOLKIT = 'communication'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+  private api_key: string | null
+  private model: string
+  private readonly network: Network
+
+  // Popular Cerebras-hosted models (override with full model IDs if needed)
+  private readonly popular_models = {
+    'zai-glm-4.7': 'zai-glm-4.7',
+    'qwen-3-235b-a22b-instruct-2507': 'qwen-3-235b-a22b-instruct-2507',
+    'qwen-3-32b': 'qwen-3-32b'
+  }
+
+  constructor(apiKey?: string) {
+    super()
+    // Load configuration from central toolkits directory
+    const toolConfigName = this.constructor.name
+      .toLowerCase()
+      .replace('tool', '')
+    this.config = ToolkitConfig.load(CerebrasTool.TOOLKIT, toolConfigName)
+
+    const toolSettings = ToolkitConfig.loadToolSettings(
+      CerebrasTool.TOOLKIT,
+      toolConfigName
+    )
+
+    // Priority: skill-provided apiKey > toolkit settings > hardcoded default
+    this.api_key =
+      apiKey || (toolSettings['CEREBRAS_API_KEY'] as string) || CEREBRAS_API_KEY
+
+    // Load model from toolkit settings or hardcoded default
+    this.model = (toolSettings['CEREBRAS_MODEL'] as string) || CEREBRAS_MODEL
+
+    this.network = new Network({ baseURL: 'https://api.cerebras.ai/v1' })
+  }
+
+  get toolName(): string {
+    return this.constructor.name
+  }
+
+  get toolkit(): string {
+    return CerebrasTool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Set the Cerebras API key
+   */
+  setApiKey(apiKey: string): void {
+    this.api_key = apiKey
+  }
+
+  /**
+   * Get list of popular available models
+   */
+  getAvailableModels(): string[] {
+    return Object.keys(this.popular_models)
+  }
+
+  /**
+   * Convert friendly model name to Cerebras model ID
+   */
+  getModelId(modelName: string): string {
+    return (
+      this.popular_models[modelName as keyof typeof this.popular_models] ||
+      modelName
+    )
+  }
+
+  /**
+   * Send a chat completion request to Cerebras
+   */
+  async chatCompletion(options: ChatCompletionOptions): Promise<ApiResponse> {
+    const {
+      messages,
+      model,
+      temperature = 0.7,
+      max_tokens,
+      system_prompt,
+      use_structured_output = false,
+      json_schema
+    } = options
+
+    if (!this.api_key) {
+      return {
+        success: false,
+        error: 'Cerebras API key not configured'
+      }
+    }
+
+    // Use default model if none provided
+    const finalModel = model || this.model
+    const modelId = this.getModelId(finalModel)
+
+    const requestMessages = []
+    if (system_prompt) {
+      requestMessages.push({ role: 'system', content: system_prompt })
+    }
+    requestMessages.push(...messages)
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const payload: any = {
+      model: modelId,
+      messages: requestMessages,
+      temperature
+    }
+
+    if (max_tokens) {
+      payload.max_tokens = max_tokens
+    }
+
+    if (use_structured_output) {
+      payload.response_format = { type: 'json_object' }
+      if (json_schema) {
+        const schemaText = JSON.stringify(json_schema)
+        const schemaPrompt = `You must return a valid JSON object that matches this schema:\n${schemaText}`
+        payload.messages = [
+          { role: 'system', content: schemaPrompt },
+          ...requestMessages
+        ]
+      }
+    }
+
+    try {
+      const response = await this.network.request({
+        url: '/chat/completions',
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.api_key}`,
+          'Content-Type': 'application/json'
+        },
+        data: payload
+      })
+
+      return {
+        success: true,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        data: response.data as any,
+        model_used: modelId
+      }
+    } catch (error: unknown) {
+      return {
+        success: false,
+        error: `Cerebras API error: ${(error as Error).message}`,
+        status_code:
+          error instanceof NetworkError ? error.response.statusCode : undefined
+      }
+    }
+  }
+
+  /**
+   * General text completion for any use case
+   */
+  async completion(options: CompletionOptions): Promise<ApiResponse> {
+    const {
+      prompt,
+      model,
+      temperature = 0.7,
+      max_tokens,
+      system_prompt,
+      use_structured_output = false,
+      json_schema
+    } = options
+
+    const messages = [{ role: 'user', content: prompt }]
+
+    const response = await this.chatCompletion({
+      messages,
+      model: model || this.model,
+      temperature,
+      max_tokens,
+      system_prompt,
+      use_structured_output,
+      json_schema
+    })
+
+    if (!response.success) {
+      return response
+    }
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const content = (response.data as any).choices[0].message.content
+
+      return {
+        success: true,
+        data: { content },
+        model_used: response.model_used
+      }
+    } catch (error: unknown) {
+      return {
+        success: false,
+        error: `Failed to extract completion: ${(error as Error).message}`
+      }
+    }
+  }
+
+  /**
+   * Generate structured JSON output using Cerebras structured outputs
+   */
+  async structuredCompletion(
+    options: StructuredCompletionOptions
+  ): Promise<ApiResponse> {
+    const {
+      prompt,
+      json_schema,
+      model,
+      temperature = 0.7,
+      max_tokens,
+      system_prompt
+    } = options
+
+    const messages = [{ role: 'user', content: prompt }]
+
+    const response = await this.chatCompletion({
+      messages,
+      model: model || this.model,
+      temperature,
+      max_tokens,
+      system_prompt,
+      use_structured_output: true,
+      json_schema
+    })
+
+    if (!response.success) {
+      return response
+    }
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const content = (response.data as any).choices[0].message.content
+      const parsedData = JSON.parse(content)
+
+      return {
+        success: true,
+        data: parsedData,
+        model_used: response.model_used
+      }
+    } catch (error: unknown) {
+      if (error instanceof SyntaxError) {
+        return {
+          success: false,
+          error: `Failed to parse JSON response: ${error.message}`
+        }
+      }
+
+      return {
+        success: false,
+        error: `Failed to extract completion: ${(error as Error).message}`
+      }
+    }
+  }
+
+  /**
+   * Get list of available models from Cerebras API
+   */
+  async listModels(): Promise<ApiResponse> {
+    if (!this.api_key) {
+      return {
+        success: false,
+        error: 'Cerebras API key not configured'
+      }
+    }
+
+    try {
+      const response = await this.network.request({
+        url: '/models',
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${this.api_key}`
+        }
+      })
+
+      return {
+        success: true,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        data: { models: (response.data as any).data }
+      }
+    } catch (error: unknown) {
+      return {
+        success: false,
+        error: `Failed to fetch models: ${(error as Error).message}`
+      }
+    }
+  }
+}

bridges/nodejs/src/sdk/tools/cerebras/index.ts

@@ -0,0 +1 @@
+export { default } from './cerebras-tool'

bridges/nodejs/src/sdk/tools/chatterbox_onnx/chatterbox_onnx-tool.ts

@@ -0,0 +1,229 @@
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import { NVIDIA_LIBS_PATH } from '@bridge/constants'
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+import { getPlatformName } from '@sdk/utils'
+
+const MODEL_NAME = 'chatterbox-multilingual-onnx'
+const DEFAULT_MAX_CHARS = 272 // Character limit to avoid hallucination
+
+interface SynthesisTask {
+  text: string
+  target_language?: string
+  audio_path: string
+  // @see https://github.com/leon-ai/leon-binaries/tree/main/bins/chatterbox_onnx/default_voices
+  voice_name?: string
+  speaker_reference_path?: string
+  cfg_strength?: number
+  exaggeration?: number
+  temperature?: number
+  // Control automatic text splitting (default: true)
+  auto_split?: boolean
+}
+
+/**
+ * Split text at natural punctuation boundaries to avoid hallucination.
+ *
+ * This function ensures no text segment exceeds maxChars by breaking at
+ * punctuation marks when possible, falling back to spaces or forced splits.
+ *
+ * @param text The text to split
+ * @param maxChars Maximum characters per segment (default: 272)
+ * @returns Array of text chunks split at natural boundaries
+ */
+function splitTextAtPunctuation(
+  text: string,
+  maxChars: number = DEFAULT_MAX_CHARS
+): string[] {
+  const trimmedText = text.trim()
+  if (trimmedText.length <= maxChars) {
+    return [trimmedText]
+  }
+
+  const chunks: string[] = []
+  let remaining = trimmedText
+
+  while (remaining.length > maxChars) {
+    // Get segment up to maxChars
+    const segment = remaining.substring(0, maxChars + 1)
+
+    // Look for punctuation followed by space (natural break)
+    const punctuationPattern = /[.!?,;:]\s/g
+    let lastMatch = -1
+    let match: RegExpExecArray | null
+
+    while ((match = punctuationPattern.exec(segment)) !== null) {
+      lastMatch = match.index + 1 // Include the punctuation but not the space
+    }
+
+    // Check if we found punctuation in a reasonable position (latter half)
+    if (lastMatch > maxChars * 0.5) {
+      chunks.push(remaining.substring(0, lastMatch).trim())
+      remaining = remaining.substring(lastMatch).trim()
+      continue
+    }
+
+    // No good punctuation found, look for last space
+    const lastSpace = segment.substring(0, maxChars).lastIndexOf(' ')
+    if (lastSpace > maxChars * 0.3) {
+      chunks.push(remaining.substring(0, lastSpace).trim())
+      remaining = remaining.substring(lastSpace).trim()
+    } else {
+      // Force split at maxChars
+      chunks.push(remaining.substring(0, maxChars).trim())
+      remaining = remaining.substring(maxChars).trim()
+    }
+  }
+
+  if (remaining.length > 0) {
+    chunks.push(remaining.trim())
+  }
+
+  return chunks
+}
+
+export default class ChatterboxONNXTool extends Tool {
+  private static readonly TOOLKIT = 'music_audio'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+
+  constructor() {
+    super()
+    // Load configuration from central toolkits directory
+    this.config = ToolkitConfig.load(ChatterboxONNXTool.TOOLKIT, this.toolName)
+  }
+
+  get toolName(): string {
+    // Use the actual config name for toolkit lookup
+    return 'chatterbox_onnx'
+  }
+
+  get toolkit(): string {
+    return ChatterboxONNXTool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Synthesize speech from text using Chatterbox ONNX
+   *
+   * By default, automatically splits long text (>272 chars) at punctuation boundaries
+   * to prevent hallucination. Split segments generate separate audio files with
+   * _part_N suffixes (e.g., output_part_0.wav, output_part_1.wav).
+   *
+   * @param tasks Array of synthesis tasks or a single task
+   * @param cudaRuntimePath Optional path to CUDA runtime for GPU acceleration (auto-detected if not provided)
+   * @returns A promise that resolves with the list of processed tasks (may include split tasks)
+   */
+  async synthesizeSpeechToFiles(
+    tasks: SynthesisTask | SynthesisTask[],
+    cudaRuntimePath?: string
+  ): Promise<Omit<SynthesisTask, 'auto_split'>[]> {
+    try {
+      // Normalize tasks to array
+      const taskArray = Array.isArray(tasks) ? tasks : [tasks]
+
+      // Process tasks: split long text into multiple tasks with _part_N suffixes
+      const tasksToSynthesize: Omit<SynthesisTask, 'auto_split'>[] = []
+
+      for (const task of taskArray) {
+        const autoSplit = task.auto_split !== undefined ? task.auto_split : true // Default: enabled
+        const text = task.text.trim()
+        const maxChars = DEFAULT_MAX_CHARS
+
+        // If auto_split disabled or text is short, pass through as-is
+        if (!autoSplit || text.length <= maxChars) {
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const { auto_split, ...cleanTask } = task
+          tasksToSynthesize.push(cleanTask)
+          continue
+        }
+
+        // Split long text at punctuation boundaries
+        const textChunks = splitTextAtPunctuation(text, maxChars)
+
+        // If only one chunk after splitting, no need for special handling
+        if (textChunks.length === 1) {
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const { auto_split, ...cleanTask } = task
+          tasksToSynthesize.push(cleanTask)
+          continue
+        }
+
+        // Multiple chunks: create separate tasks with _part_N suffixes
+        const audioPath = task.audio_path
+        const parsedPath = path.parse(audioPath)
+        const basePath = path.join(parsedPath.dir, parsedPath.name)
+        const ext = parsedPath.ext
+
+        for (let i = 0; i < textChunks.length; i += 1) {
+          const chunk = textChunks[i]
+          if (!chunk) continue
+
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const {
+            auto_split,
+            text: _text,
+            audio_path: _audioPath,
+            ...baseTask
+          } = task
+          tasksToSynthesize.push({
+            ...baseTask,
+            text: chunk,
+            audio_path: `${basePath}_part_${i}${ext}`
+          })
+        }
+      }
+
+      // Get model path using the generic resource system
+      const modelPath = await this.getResourcePath(MODEL_NAME)
+
+      // Create a temporary JSON file for the tasks
+      const tempDir = await fs.promises.mkdtemp(
+        path.join(os.tmpdir(), 'chatterbox_onnx_tasks_')
+      )
+      const jsonFilePath = path.join(tempDir, 'tasks.json')
+
+      await fs.promises.writeFile(
+        jsonFilePath,
+        JSON.stringify(tasksToSynthesize, null, 2),
+        'utf8'
+      )
+
+      const args = [
+        '--function',
+        'synthesize_speech',
+        '--json_file',
+        jsonFilePath,
+        '--resource_path',
+        modelPath
+      ]
+
+      // Auto-detect CUDA runtime path if not provided
+      const platformName = getPlatformName()
+      const shouldUseCuda =
+        platformName === 'linux-x86_64' || platformName === 'win-amd64'
+      const finalCudaRuntimePath =
+        cudaRuntimePath ?? (shouldUseCuda ? NVIDIA_LIBS_PATH : undefined)
+
+      if (finalCudaRuntimePath) {
+        args.push('--cuda_runtime_path', finalCudaRuntimePath)
+      }
+
+      await this.executeCommand({
+        binaryName: 'chatterbox_onnx',
+        args,
+        options: { sync: true }
+      })
+
+      // Return the processed tasks so caller knows which files were created
+      return tasksToSynthesize
+    } catch (error: unknown) {
+      throw new Error(`Speech synthesis failed: ${(error as Error).message}`)
+    }
+  }
+}

bridges/nodejs/src/sdk/tools/chatterbox_onnx/index.ts

@@ -0,0 +1 @@
+export { default } from './chatterbox_onnx-tool'

bridges/nodejs/src/sdk/tools/ecapa/ecapa-tool.ts

@@ -0,0 +1,102 @@
+import fs from 'node:fs'
+
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+
+const MODEL_NAME = 'ecapa-voice_gender_classifier'
+
+export default class ECAPATool extends Tool {
+  private static readonly TOOLKIT = 'music_audio'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+
+  constructor() {
+    super()
+    // Load configuration from central toolkits directory
+    this.config = ToolkitConfig.load(ECAPATool.TOOLKIT, this.toolName)
+  }
+
+  get toolName(): string {
+    // Use the actual config name for toolkit lookup
+    return 'ecapa'
+  }
+
+  get toolkit(): string {
+    return ECAPATool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Detect gender from audio file using ECAPA-TDNN voice gender classifier
+   * @param inputPath The file path of the audio to be analyzed
+   * @param device Device to use for processing (cpu, cuda)
+   * @returns A promise that resolves with the detected gender: "male", "female", or "unknown"
+   */
+  async detectGender(inputPath: string, device = 'cpu'): Promise<string> {
+    try {
+      // Validate input file exists
+      if (!fs.existsSync(inputPath)) {
+        throw new Error(`Input file does not exist: ${inputPath}`)
+      }
+
+      // Get model path using the generic resource system
+      const modelPath = await this.getResourcePath(MODEL_NAME)
+
+      const args = [
+        '--function',
+        'detect_gender',
+        '--input',
+        inputPath,
+        '--model_path',
+        modelPath,
+        '--device',
+        device
+      ]
+
+      const result = await this.executeCommand({
+        binaryName: 'ecapa-voice_gender_classifier',
+        args,
+        options: { sync: true }
+      })
+
+      // Parse the output to extract gender
+      const gender = this.parseGenderOutput(result)
+
+      return gender
+    } catch (error: unknown) {
+      throw new Error(
+        `Voice gender detection failed: ${(error as Error).message}`
+      )
+    }
+  }
+
+  /**
+   * Parse the gender detection output
+   */
+  private parseGenderOutput(rawOutput: string): string {
+    const lines = rawOutput.split('\n')
+
+    // Look for gender result in the output
+    for (const line of lines) {
+      const lowerLine = line.toLowerCase().trim()
+
+      if (lowerLine.includes('gender:')) {
+        // Extract gender from line like "Gender: male"
+        const match = lowerLine.match(/gender:\s*(male|female|unknown)/i)
+        if (match && match[1]) {
+          return match[1].toLowerCase()
+        }
+      }
+
+      // Also check for direct gender output
+      if (lowerLine === 'male' || lowerLine === 'female') {
+        return lowerLine
+      }
+    }
+
+    // If no clear gender found, return unknown
+    return 'unknown'
+  }
+}

bridges/nodejs/src/sdk/tools/ecapa/index.ts

@@ -0,0 +1 @@
+export { default } from './ecapa-tool'

bridges/nodejs/src/sdk/tools/elevenlabs_audio/elevenlabs_audio-tool.ts

@@ -0,0 +1,270 @@
+import fs from 'node:fs'
+
+import FormData from 'form-data'
+
+import type { TranscriptionOutput } from '@sdk/tools/transcription-schema'
+import { Tool } from '@sdk/base-tool'
+import { ToolkitConfig } from '@sdk/toolkit-config'
+import { Network } from '@sdk/network'
+
+// Hardcoded default settings for ElevenLabs audio tool
+// These can be overridden by toolkit settings.json per toolkit.
+const ELEVENLABS_AUDIO_API_KEY: string | null = null
+const ELEVENLABS_AUDIO_MODEL = 'scribe_v1'
+
+interface ElevenLabsWord {
+  text: string
+  start: number
+  end: number
+  type: 'word' | 'spacing' | 'audio_event'
+  speaker_id?: string
+}
+
+interface ElevenLabsTranscriptionResponse {
+  language_code: string
+  language_probability: number
+  text: string
+  words: ElevenLabsWord[]
+}
+
+interface ElevenLabsDubbingCreateResponse {
+  dubbing_id: string
+  expected_duration_sec: number
+}
+
+interface ElevenLabsDubbingStatusResponse {
+  dubbing_id: string
+  name: string
+  status: 'dubbing' | 'dubbed' | 'failed'
+  target_languages: string[]
+  error?: string | null
+  created_at?: string
+  editable?: boolean | null
+}
+
+export default class ElevenLabsAudioTool extends Tool {
+  private static readonly TOOLKIT = 'music_audio'
+  private readonly config: ReturnType<typeof ToolkitConfig.load>
+  readonly apiKey: string | null
+  readonly model: string
+
+  constructor() {
+    super()
+    this.config = ToolkitConfig.load(ElevenLabsAudioTool.TOOLKIT, this.toolName)
+
+    const toolSettings = ToolkitConfig.loadToolSettings(
+      ElevenLabsAudioTool.TOOLKIT,
+      this.toolName
+    )
+
+    // Priority: toolkit settings > hardcoded default
+    this.apiKey =
+      (toolSettings['ELEVENLABS_AUDIO_API_KEY'] as string) ||
+      ELEVENLABS_AUDIO_API_KEY
+    this.model =
+      (toolSettings['ELEVENLABS_AUDIO_MODEL'] as string) ||
+      ELEVENLABS_AUDIO_MODEL
+  }
+
+  get toolName(): string {
+    return 'elevenlabs_audio'
+  }
+
+  get toolkit(): string {
+    return ElevenLabsAudioTool.TOOLKIT
+  }
+
+  get description(): string {
+    return this.config['description']
+  }
+
+  /**
+   * Transcribe audio to a file using ElevenLabs' Scribe v1 API
+   * @param inputPath Path to the audio file to transcribe
+   * @param outputPath Path to save the JSON transcription (unified format)
+   * @param apiKey ElevenLabs API key (uses env/hardcoded default if not provided)
+   * @param model Transcription model (defaults to tool default)
+   * @param diarize Whether to enable speaker diarization (defaults to true)
+   */
+  async transcribeToFile(
+    inputPath: string,
+    outputPath: string,
+    apiKey?: string,
+    model?: string,
+    diarize = true
+  ): Promise<string> {
+    // Use provided values, instance values, or error
+    const finalApiKey = apiKey || this.apiKey
+    const finalModel = model || this.model
+    if (!finalApiKey) {
+      throw new Error('ElevenLabs API key is missing')
+    }
+
+    const form = new FormData()
+    form.append('file', fs.createReadStream(inputPath))
+    form.append('model_id', finalModel)
+    form.append('diarize', diarize.toString())
+    form.append('tag_audio_events', 'true')
+    form.append('timestamps_granularity', 'word')
+
+    const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
+    const response = await network.request<ElevenLabsTranscriptionResponse>({
+      url: '/v1/speech-to-text',
+      method: 'POST',
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      data: form as any,
+      headers: {
+        'xi-api-key': apiKey,
+        ...form.getHeaders()
+      }
+    })
+
+    const normalizedOutput: TranscriptionOutput = this.parseTranscription(
+      response.data
+    )
+
+    await fs.promises.writeFile(
+      outputPath,
+      JSON.stringify(normalizedOutput, null, 2),
+      'utf8'
+    )
+
+    return outputPath
+  }
+
+  /**
+   * Create a dubbing project using ElevenLabs' Dubbing API
+   * @param inputPath Path to the audio/video file to dub
+   * @param targetLang Target language code (e.g., 'es', 'fr', 'zh')
+   * @param apiKey ElevenLabs API key
+   * @param sourceLang Source language code (defaults to 'auto')
+   * @param numSpeakers Number of speakers (0 for auto-detect)
+   * @param watermark Whether to add watermark to output video
+   * @returns Dubbing project ID and expected duration
+   */
+  async createDubbing(
+    inputPath: string,
+    targetLang: string,
+    apiKey: string,
+    sourceLang = 'auto',
+    numSpeakers = 0,
+    watermark = false
+  ): Promise<ElevenLabsDubbingCreateResponse> {
+    if (!apiKey) {
+      throw new Error('ElevenLabs API key is missing')
+    }
+
+    const form = new FormData()
+    form.append('file', fs.createReadStream(inputPath))
+    form.append('target_lang', targetLang)
+    form.append('source_lang', sourceLang)
+    form.append('num_speakers', numSpeakers.toString())
+    form.append('watermark', watermark.toString())
+
+    const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
+    const response = await network.request<ElevenLabsDubbingCreateResponse>({
+      url: '/v1/dubbing',
+      method: 'POST',
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      data: form as any,
+      headers: {
+        'xi-api-key': apiKey,
+        ...form.getHeaders()
+      }
+    })
+
+    return response.data
+  }
+
+  /**
+   * Get the status of a dubbing project
+   * @param dubbingId The dubbing project ID
+   * @param apiKey ElevenLabs API key
+   * @returns Dubbing project status information
+   */
+  async getDubbingStatus(
+    dubbingId: string,
+    apiKey: string
+  ): Promise<ElevenLabsDubbingStatusResponse> {
+    if (!apiKey) {
+      throw new Error('ElevenLabs API key is missing')
+    }
+
+    const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
+    const response = await network.request<ElevenLabsDubbingStatusResponse>({
+      url: `/v1/dubbing/${dubbingId}`,
+      method: 'GET',
+      headers: {
+        'xi-api-key': apiKey
+      }
+    })
+
+    return response.data
+  }
+
+  /**
+   * Download the dubbed file
+   * @param dubbingId The dubbing project ID
+   * @param targetLang Target language code
+   * @param outputPath Path to save the dubbed file
+   * @param apiKey ElevenLabs API key
+   * @returns Path to the downloaded file
+   */
+  async downloadDubbedFile(
+    dubbingId: string,
+    targetLang: string,
+    outputPath: string,
+    apiKey: string
+  ): Promise<string> {
+    if (!apiKey) {
+      throw new Error('ElevenLabs API key is missing')
+    }
+
+    const network = new Network({ baseURL: 'https://api.elevenlabs.io' })
+    const response = await network.request({
+      url: `/v1/dubbing/${dubbingId}/audio/${targetLang}`,
+      method: 'GET',
+      headers: {
+        'xi-api-key': apiKey
+      },
+      responseType: 'arraybuffer'
+    })
+
+    // Write the audio/video file
+    await fs.promises.writeFile(
+      outputPath,
+      Buffer.from(response.data as ArrayBuffer)
+    )
+
+    return outputPath
+  }
+
+  private parseTranscription(
+    rawOutput: ElevenLabsTranscriptionResponse
+  ): TranscriptionOutput {
+    const wordItems = rawOutput.words.filter((item) => item.type === 'word')
+    const uniqueSpeakers = Array.from(
+      new Set(wordItems.map((word) => word.speaker_id).filter(Boolean))
+    ) as string[]
+
+    // Calculate duration from the last word's end time
+    const duration =
+      wordItems.length > 0 ? wordItems[wordItems.length - 1]?.end : 0
+    const segments = wordItems.map((word) => ({
+      from: word.start,
+      to: word.end,
+      text: word.text,
+      speaker: word.speaker_id || null
+    }))
+
+    return {
+      duration: duration ?? 0,
+      speakers: uniqueSpeakers,
+      speaker_count: uniqueSpeakers.length,
+      segments,
+      metadata: {
+        tool: this.toolName
+      }
+    }
+  }
+}