Merge branch 'develop' into feat/akinator-rewrite

We might in the future generate automatically from the TypeScript types, generate the Python types.
For the moment, it is not typed, so we use a dict.

.dockerignore

@@ -1,9 +0,0 @@
-__pycache__/
-**/dist/*
-**/build/
-**/node_modules/
-**/tmp/*
-**/src/.venv/*
-logs/*
-*.pyc
-.DS_Store

.env.sample

@@ -8,6 +8,17 @@ LEON_LANG=en-US
 LEON_HOST=http://localhost
 LEON_PORT=1337
 
+# Enable/disable LLM
+LEON_LLM=false
+# LLM provider
+LEON_LLM_PROVIDER=local
+# LLM provider API key (if not local)
+LEON_LLM_PROVIDER_API_KEY=
+# Enable/disable LLM natural language generation
+LEON_LLM_NLG=false
+# Enable/disable LLM Action Recognition
+LEON_LLM_ACTION_RECOGNITION=false
+
 # Time zone (current one by default)
 LEON_TIME_ZONE=
 
@@ -39,7 +50,7 @@ LEON_PY_TCP_SERVER_HOST=0.0.0.0
 LEON_PY_TCP_SERVER_PORT=1342
 
 # Path to the Pipfile
-PIPENV_PIPFILE=bridges/python/src/Pipfile
+PIPENV_PIPFILE=tcp_server/src/Pipfile
 
 # Path to the virtual env in .venv/
 PIPENV_VENV_IN_PROJECT=true

.eslintrc.json

@@ -18,10 +18,7 @@
     "browser": true
   },
   "parserOptions": {
-    "ecmaVersion": 2021
-  },
-  "globals": {
-    "io": true
+    "ecmaVersion": "latest"
   },
   "plugins": ["@typescript-eslint", "unicorn", "import"],
   "ignorePatterns": "*.spec.js",
@@ -29,7 +26,7 @@
     "@typescript-eslint/no-non-null-assertion": ["off"],
     "no-async-promise-executor": ["off"],
     "no-underscore-dangle": ["error", { "allowAfterThis": true }],
-    "prefer-destructuring": ["error"],
+    "prefer-destructuring": ["off"],
     "comma-dangle": ["error", "never"],
     "semi": ["error", "never"],
     "object-curly-spacing": ["error", "always"],

.github/CONTRIBUTING.md

@@ -77,22 +77,6 @@ npm run dev:server
 npm run dev:app
 ```
 
-### Docker
-
-```sh
-# Clone the repository
-git clone https://github.com/leon-ai/leon.git leon
-
-# Go to the project root
-cd leon
-
-# Build
-npm run docker:build
-
-# Run the development server and the development web app
-npm run docker:dev
-```
-
 ## Versioning
 
 - We use [Semantic Versioning](https://semver.org) for releases.
@@ -125,7 +109,6 @@ Scopes define high-level nodes of Leon.
 
 - bridge/python
 - bridge/nodejs
-- docker
 - hotword
 - scripts
 - server
@@ -142,6 +125,11 @@ git commit -m "chore: split training script into awesome blocks"
 git commit -m "style(web app): remove chatbot useless parentheses"
 ```
 
+### GUI Clients
+
+If you are using a GUI client such as GitKraken, you may need to disable the default Git executable to make sure to use your default shell.
+Otherwise you may encounter an error such as "npx not found".
+
 ## Sponsor
 
 You can also contribute by [sponsoring Leon](http://sponsor.getleon.ai).
@@ -190,11 +178,11 @@ Once Pyenv installed, run:
 
 ```bash
 # Install Python
-pyenv install 3.9.10 --force
-pyenv global 3.9.10
+pyenv install 3.11.9 --force
+pyenv global 3.11.9
 
 # Install Pipenv
-pip install pipenv==2022.7.24
+pip install pipenv==2024.0.1
 ```
 
 Your Python environment should be ready now. So now you can set up the respective environments according to what you are going to contribute to and build them:

.github/ISSUE_TEMPLATE/BUG.md

@@ -16,7 +16,6 @@ If the bug is related to the setup, please submit the issue at: https://github.c
 - OS (or browser) version:
 - Node.js version:
 - Complete "leon check" (or "npm run check") output:
-- (if using Docker) Complete "npm run docker:check" output:
 - (optional) Leon skill version:
 
 ### Expected Behavior

.github/dependabot.yml

@@ -1,13 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: 'npm'
-    directory: '/'
-    schedule:
-      interval: 'weekly'
-      day: 'friday'
-      time: '22:00'
-    commit-message:
-      prefix: 'chore'
-      include: 'scope'
-    reviewers:
-      - 'louistiti'

.github/workflows/pre-release-python-bridge.yml

@@ -13,7 +13,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        # @see https://github.com/actions/runner-images/tree/main/images/macos
+        # Use macos-12 instead of macos-latest because the latter use ARM64 (M1) architecture
+        os: [ubuntu-20.04, macos-12]
+        # Temporarily disable Windows release
+        # os: [ubuntu-20.04, macos-12, windows-latest]
 
     runs-on: ${{ matrix.os }}
 
@@ -24,10 +28,10 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Install Pipenv
-        run: pip install --upgrade pip && pip install pipenv==2022.7.24
+        run: pip install --upgrade pip && pip install pipenv==2024.0.1
 
       - name: Install Node.js
         uses: actions/setup-node@v3
@@ -69,7 +73,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Set Python bridge version
         working-directory: bridges/python/src

.github/workflows/pre-release-tcp-server.yml

@@ -13,7 +13,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
+        # @see https://github.com/actions/runner-images/tree/main/images/macos
+        # Use macos-12 instead of macos-latest because the latter use ARM64 (M1) architecture
+        os: [ubuntu-20.04, macos-12, windows-latest]
 
     runs-on: ${{ matrix.os }}
 
@@ -24,10 +26,10 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Install Pipenv
-        run: pip install --upgrade pip && pip install pipenv==2022.7.24
+        run: pip install --upgrade pip && pip install pipenv==2024.0.1
 
       - name: Install Node.js
         uses: actions/setup-node@v3
@@ -69,7 +71,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9.10
+          python-version: 3.11.9
 
       - name: Set TCP server version
         working-directory: tcp_server/src

.gitignore

@@ -31,6 +31,12 @@ tcp_server/src/Pipfile.lock
 skills/**/src/settings.json
 skills/**/memory/*.json
 core/data/models/*.nlp
+core/data/models/*.json
+core/data/models/llm/*
+core/data/models/audio/tts/**/*.*
+!core/data/models/audio/tts/config.json
+core/data/models/audio/asr/**/*.*
+!core/data/models/**/.gitkeep
 package.json.backup
 .python-version
 schemas/**/*.json

.husky/commit-msg

@@ -6,6 +6,6 @@ if ! [ -x "$(command -v npm)" ]; then
   echo "If you use a version manager tool such as nvm and a git GUI such as GitKraken, please read: https://typicode.github.io/husky/#/?id=command-not-found" >&2
   exit 1
 else
-  npx ts-node scripts/commit-msg.js
+  npx tsx scripts/commit-msg.js
 fi
 

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-npx lint-staged
+npm run pre-commit

Dockerfile

@@ -1,60 +0,0 @@
-FROM ubuntu:20.04
-ENV IS_DOCKER true
-
-# Replace shell with bash so we can source files
-RUN rm /bin/sh && ln -s /bin/bash /bin/sh
-
-# Set debconf to run non-interactively
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-# Install base dependencies
-RUN apt-get update && apt-get install --yes -q --no-install-recommends \
-  apt-transport-https \
-  build-essential \
-  ca-certificates \
-  curl \
-  git \
-  wget \
-  libssl-dev \
-  openssl \
-  libz-dev \
-  zlib1g-dev \
-  libbz2-dev \
-  libreadline-dev \
-  libsqlite3-dev \
-  llvm \
-  libncurses5-dev \
-  xz-utils \
-  tk-dev libxml2-dev \
-  libxmlsec1-dev \
-  libffi-dev \
-  liblzma-dev \
-  libgdbm-dev \
-  libnss3-dev \
-  libc6-dev
-
-# Run the container as an unprivileged user
-RUN groupadd docker && useradd -g docker -s /bin/bash -m docker
-USER docker
-WORKDIR /home/docker
-
-# Install Node.js with nvm
-ENV NVM_DIR /home/docker/.nvm
-ENV NODE_VERSION v16.18.0
-
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash
-RUN /bin/bash -c "source $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use --delete-prefix $NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules
-ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH
-
-# Install Leon
-WORKDIR /home/docker/leon
-USER root
-RUN chown -R docker /home/docker/leon
-USER docker
-COPY --chown=docker ./ ./
-RUN npm install
-RUN npm run build
-
-CMD ["npm", "start"]

README.md

@@ -30,38 +30,41 @@ _<p align="center">Your open-source personal assistant.</p>_
 
 ---
 
-## Current State
+## Important Notice (as of 2024-06-18)
+
+> [!IMPORTANT]
+> Due to all the new major changes coming to Leon AI, the development branch might be unstable. It is recommended to use the older version under the master branch.
+>
+> Please note that older versions do not make use of any foundation model, which will be introduced in upcoming versions.
+
+**Outdated Documentation**
+
+Please note that the documentation and this README are not up to date. We've made significant changes to Leon over the past few months, including the introduction of new TTS and ASR engines, and a hybrid approach that balances LLM, simple classification, and multiple NLP techniques to achieve optimal speed, customization, and accuracy. We'll update the documentation for the official release.
+
+**Project History and Future Plans**
+
+Since its inception in 2017, Leon has undergone significant transformations. Although we've been inconsistent in shipping updates over the years, we're now focused on maturing the project. With the recent integration of transformers-based models, we're prepared to unlock Leon's full potential.
+
+Our next step is to finalize the latest features for the official release.
+Then we'll be establishing a group of active contributors to work together, develop new skills, and share them with the community. A skill registry platform will be built (see it as the npm or pip registry but for skills).
+
+- Check out [the roadmap](http://roadmap.getleon.ai/) for more information on our upcoming plans.
+- Watch a [preview of our last progress](https://www.youtube.com/watch?v=6CInSt6pTVA) to see what we've been working on.
+
+**Get Involved**
+
+[Join us on Discord](https://discord.gg/MNQqqKg) to ask questions, or express interest in becoming an active contributor.
+
+---
 
 ### Why is there a small amount of contributors?
 
-I'm taking a lot of time to work on the new core of Leon due to personal reasons. I can only work on it after work and on weekends. Hence, **I'm blocking any potential contribution as the whole core of Leon is coming with many breaking changes**. Many of you are willing to contribute in Leon (create new skills, help to improve the core, translations and so on...), a big thanks to every one of you!
+I'm taking a lot of time to work on the new core of Leon due to personal reasons. I can only work on it during my spare time. Hence, I'm blocking any contribution as the whole core of Leon is coming with many breaking changes. Many of you are willing to contribute in Leon (create new skills, help to improve the core, translations and so on...), a big thanks to every one of you!
 
 While I would love to devote more time to Leon, I'm currently unable to do so because I have bills to pay. I have some ideas about how to monetize Leon in the future (Leon's core will always remain open source), but before to get there there is still a long way to go.
 
 Until then, any financial support by [sponsoring Leon](http://sponsor.getleon.ai) is much appreciated 🙂
 
-### How about large language models and Leon?
-
-Since AI gained in popularity and large language models are getting more and more traction, many of you joined our community. A huge welcome to all of you! 🤗
-
-At the moment, Leon's NLU will remain intents first with his own model without relying on an LLM. It is important that Leon can run 100% offline and I'm confident that with the downsizing techniques such as quantization Leon will sooner or later work with LLMs at his core and still be able to run on edge.
-
-Here is how LLMs may help Leon in the future:
-
-- Intent fallback: when an utterance cannot match an intent, then rely on an LLM to provide results.
-- New named entity recognition engine: provide a better solution to extract entities from utterances such as fruits, numbers, cities, durations, persons, etc.
-- Skill features: let skills leverage LLMs to provide out-of-the-box NLP features such as summarization, knowledge base, translation, sentiment analysis and so on...
-- Skill building: LLMs can help to develop skills such as paraphrasing utterance samples, translate answers, convert code from our Python bridge to the upcoming JavaScript bridge and vice versa, etc.
-- More...
-
-### What's Next?
-
-Once the new core released, we'll work on the community aspect of Leon. For example, better organize [our Discord](https://discord.gg/MNQqqKg), planify regular calls, work on skills together, etc. It is very important for Leon to have a real community. At that moment, the skills platform will already be online, so it'll be easier to sync our progress and publish new skills.
-
-- Feel free to check out the Git development branches and our [next major milestones](https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/#whats-next).
-- And the [detailed roadmap](http://roadmap.getleon.ai).
-- Many exciting things are coming up, hence no new documentation and test are going to be written until the official release of Leon.
-
 ---
 
 ## Latest Release
@@ -152,19 +155,6 @@ leon start
 # Hooray! Leon is running
 ```
 
-### Docker Installation
-
-```sh
-# Install Leon
-leon create birth --docker
-
-# Run
-leon start
-
-# Go to http://localhost:1337
-# Hooray! Leon is running
-```
-
 ## 📚 Documentation
 
 For full documentation, visit [docs.getleon.ai](https://docs.getleon.ai).

app/src/css/style.scss

@@ -1,4 +1,11 @@
-@import url(https://fonts.googleapis.com/css?family=Open+Sans:400,600,700,800);
+@import '@fontsource/source-sans-pro/200.css';
+@import '@fontsource/source-sans-pro/300.css';
+@import '@fontsource/source-sans-pro/400.css';
+@import '@fontsource/source-sans-pro/600.css';
+@import '@fontsource/source-sans-pro/700.css';
+@import '@fontsource/source-sans-pro/900.css';
+@import 'remixicon/fonts/remixicon.css';
+@import 'voice-energy/main.scss';
 
 html,
 body,
@@ -83,7 +90,6 @@ video {
   padding: 0;
   border: 0;
   font-size: 100%;
-  font: inherit;
   vertical-align: baseline;
 }
 
@@ -129,48 +135,68 @@ table {
   --light-black-color: #222426;
   --white-color: #fff;
   --grey-color: #323739;
+  --blue-color: #1c75db;
+  --pink-color: #ed297a;
+
+  .settingup {
+    --a-loader-size-md: 20px !important;
+  }
 }
 
 a {
   color: inherit;
 }
 
-ul li {
+#feed ul li:not(.aurora-list-item) {
   margin-left: 20px;
 }
 
 body {
   color: var(--white-color);
   background-color: var(--black-color);
-  font-family: 'Open Sans', sans-serif;
+  font-family:
+    'Source Sans Pro',
+    system-ui,
+    -apple-system,
+    BlinkMacSystemFont,
+    'Segoe UI',
+    Roboto,
+    'Helvetica Neue',
+    Arial,
+    'Noto Sans',
+    sans-serif,
+    'Apple Color Emoji',
+    'Segoe UI Emoji',
+    'Segoe UI Symbol',
+    'Noto Color Emoji';
   font-weight: 400;
 }
 body > * {
   transition: opacity 0.5s;
 }
-body.settingup > * {
+body.settingup > *:not(#init) {
   opacity: 0;
 }
-body.settingup::after {
-  position: absolute;
-  content: '';
-  width: 32px;
-  height: 32px;
-  background-color: #777;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
-  border-radius: 50%;
-  animation: scaleout 0.6s infinite ease-in-out;
+#init .not-initialized {
+  visibility: hidden;
 }
-@keyframes scaleout {
-  0% {
-    transform: scale(0);
-  }
-  100% {
-    transform: scale(1);
-    opacity: 0;
-  }
+#init .initialized {
+  opacity: 0;
+  visibility: hidden;
+}
+
+kbd {
+  font-family: 'Source Sans Pro', monospace;
+  display: inline-block;
+  background-color: var(--light-black-color);
+  color: rgba(255, 255, 255, 0.4);
+  border-radius: 4px;
+  text-align: center;
+  min-width: 16px;
+  min-height: 16px;
+  line-height: 16px !important;
+  padding: 2px 6px !important;
+  margin: 0 !important;
 }
 
 main {
@@ -187,24 +213,43 @@ footer {
   text-align: center;
   left: 50%;
   bottom: 0;
+  line-height: 18px;
   transform: translate(-50%, -50%);
 }
 
-input {
+textarea {
+  font-family: inherit;
   text-align: center;
   color: var(--white-color);
   width: 100%;
   border: none;
   border-bottom: 2px solid var(--grey-color);
   background: none;
-  font-weight: 400;
+  font-weight: 600;
   font-size: 4em;
   padding-right: 39px;
+  height: 140px;
+  resize: none;
+  overflow-y: auto;
+}
+textarea::-webkit-scrollbar {
+  width: 6px;
+}
+textarea::-webkit-scrollbar-thumb {
+  background-color: rgba(255, 255, 255, 0.2);
+  border-radius: 12px;
 }
 
-small {
+#tip {
+  display: inline-flex;
+  margin-top: 2px;
   color: var(--white-color);
-  font-size: 0.7em;
+  line-height: 22px;
+  font-size: 0.9em;
+  gap: 20px;
+  li {
+    margin-left: 0;
+  }
 }
 
 .hide {
@@ -218,10 +263,32 @@ small {
   height: 40px;
 }
 
+#top-container {
+  position: absolute;
+  top: 4%;
+  color: var(--grey-color);
+  display: flex;
+  width: 100%;
+  justify-content: space-between;
+}
+#mood {
+  position: relative;
+  font-size: 16px;
+}
+#info {
+  position: relative;
+  text-decoration: underline;
+  background: none;
+  border: none;
+  color: var(--grey-color);
+  cursor: pointer;
+  font-size: inherit;
+}
+
 #feed {
   position: absolute;
   width: 100%;
-  top: 10%;
+  top: 8%;
   height: 50%;
   overflow-y: auto;
   border: 2px solid var(--grey-color);
@@ -278,6 +345,15 @@ small {
   }
 }
 
+.llm-token {
+  opacity: 0;
+  transition: opacity 2.5s;
+}
+
+.llm-token.fade-in {
+  animation: fadeIn 2.5s forwards;
+}
+
 .bubble-container {
   padding: 6px;
 }
@@ -300,7 +376,7 @@ small {
 }
 
 .bubble {
-  padding: 6px 12px;
+  padding: 10px 16px;
   border-radius: 16px;
   display: inline-block;
   max-width: 60%;
@@ -309,6 +385,8 @@ small {
   opacity: 0;
   animation: fadeIn 0.2s ease-in forwards;
   overflow: hidden;
+  font-size: 1.8rem;
+  line-height: 2.4rem;
 }
 #feed .me .bubble {
   background-color: #1c75db;
@@ -343,7 +421,9 @@ small {
   padding: 2px 8px;
   font-size: inherit;
   cursor: pointer;
-  transition: background-color 0.2s, color 0.2s;
+  transition:
+    background-color 0.2s,
+    color 0.2s;
 }
 .suggestion:hover {
   color: var(--black-color);
@@ -353,7 +433,7 @@ small {
 #input-container {
   position: absolute;
   width: 100%;
-  bottom: 22%;
+  bottom: 18%;
 }
 
 #mic-container {

app/src/css/voice-energy/base.scss

@@ -0,0 +1,162 @@
+/**
+ * Overlay and containers
+ */
+
+body.voice-mode-enabled {
+  #voice-overlay-transitor,
+  #voice-overlay-bg {
+    visibility: visible;
+  }
+  #voice-overlay-bg {
+    opacity: 1;
+  }
+}
+#voice-overlay-transitor {
+  position: fixed;
+  background-color: var(--black-color);
+  z-index: 10;
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  will-change: transform;
+  animation: scaleIn 1s;
+}
+@keyframes scaleIn {
+  0% {
+    transform: scale3d(0, 0, 1);
+  }
+  100% {
+    transform: scale3d(172, 172, 1);
+  }
+}
+
+#voice-status,
+#voice-tips {
+  color: var(--grey-color);
+  text-align: center;
+}
+#voice-status {
+  font-size: 17px;
+  font-style: italic;
+}
+#voice-tips {
+  margin-top: 32px;
+  line-height: 18px;
+  font-size: 15px;
+}
+
+#voice-overlay-bg {
+  visibility: hidden;
+  cursor: pointer;
+  opacity: 0;
+  position: fixed;
+  width: 100vw;
+  height: 100vh;
+  z-index: 100;
+  will-change: opacity;
+  display: flex;
+  justify-content: center;
+  // backdrop-filter: saturate(140%) blur(5px);
+  /*background-color: rgba(0, 0, 0, .9);*/
+  background-color: var(--black-color);
+}
+@keyframes skipFadeIn {
+  0% {
+    opacity: 1;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
+#voice-container {
+  opacity: 0;
+  position: relative;
+  top: 64px;
+  display: flex;
+  flex-direction: column;
+  width: 1024px;
+  height: 756px;
+  align-items: center;
+  gap: 64px;
+  animation: fadeIn 1s 3s both;
+}
+
+#voice-energy-container {
+  --neon-size: 228px;
+
+  // animation: fadeIn 1s 1.5s both;
+  //opacity: 0;
+  overflow: hidden;
+  position: relative;
+  height: 400px;
+  border-radius: 50%;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+p#voice-speech {
+  width: 100%;
+  height: 100%;
+  flex: 1;
+  text-align: center;
+  font-size: 3rem;
+  font-weight: 600;
+}
+
+/**
+ * Neons
+ */
+
+.voice-neon {
+  position: absolute;
+  z-index: 10;
+  width: var(--neon-size);
+  height: var(--neon-size);
+}
+
+#purple-neon-blur {
+  --neon-blur: calc(var(--neon-size) + 96px);
+
+  position: absolute;
+  z-index: 0;
+  opacity: 0.7;
+  width: var(--neon-blur);
+  height: var(--neon-blur);
+}
+
+#blue-neon-1 {
+  margin-top: -8px;
+  margin-left: 12px;
+}
+
+#blue-neon-2 {
+  margin-top: 8px;
+  margin-right: 12px;
+}
+
+/**
+ * Particles
+ */
+
+.voice-particle {
+  position: absolute;
+  width: 3px;
+  height: 3px;
+  border-radius: 50%;
+  opacity: 1;
+  will-change: transform, opacity;
+  animation-duration: 1s;
+  animation-iteration-count: infinite;
+}
+.voice-particle.blue {
+  background-color: #c4e0ff;
+  box-shadow: 0 0 2px 2px var(--blue-color);
+}
+.voice-particle.pink {
+  background-color: #ffb9d7;
+  box-shadow: 0 0 2px 2px var(--pink-color);
+}

app/src/css/voice-energy/idle.scss

@@ -0,0 +1,59 @@
+/**
+ * IDLE status
+ */
+
+#voice-energy-container.idle {
+  .voice-particle {
+    visibility: hidden;
+    animation-play-state: paused;
+    opacity: 0;
+  }
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    transform: scale(1);
+  }
+  #purple-neon-blur circle {
+    filter: drop-shadow(0px 0px 64px mix(#ed297a, #1c75db));
+    animation: idleNeonBlurBreath 2.2s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: idleBouncePinkNeon1 1.8s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: idleMoveBlueNeon1 1.8s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: idleMoveBlueNeon2 1.8s 0.5s infinite alternate;
+  }
+}
+
+@keyframes idleNeonBlurBreath {
+  100% {
+    filter: drop-shadow(0px 0px 0px mix(#ed297a, #1c75db));
+  }
+}
+@keyframes idleBouncePurpleNeonBlur {
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes idleBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes idleMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes idleMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}

app/src/css/voice-energy/listening.scss

@@ -0,0 +1,59 @@
+/**
+ * Listening status
+ */
+
+#voice-energy-container.listening {
+  .voice-particle {
+    visibility: hidden;
+    animation-play-state: paused;
+    opacity: 0;
+  }
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    transform: scale(1);
+  }
+  #purple-neon-blur circle {
+    filter: drop-shadow(0px 0px 64px mix(#ed297a, #1c75db));
+    animation: listeningNeonBlurBreath 0.7s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: listeningBouncePinkNeon1 0.3s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: listeningMoveBlueNeon1 0.3s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: listeningMoveBlueNeon2 0.3s 0.5s infinite alternate;
+  }
+}
+
+@keyframes listeningNeonBlurBreath {
+  100% {
+    filter: drop-shadow(0px 0px 0px mix(#ed297a, #1c75db));
+  }
+}
+@keyframes listeningBouncePurpleNeonBlur {
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes listeningBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes listeningMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes listeningMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}

app/src/css/voice-energy/main.scss

@@ -0,0 +1,5 @@
+@import 'base.scss';
+@import 'listening.scss';
+@import 'idle.scss';
+@import 'processing.scss';
+@import 'talking.scss';

app/src/css/voice-energy/processing.scss

@@ -0,0 +1,88 @@
+@use 'sass:math';
+
+/**
+ * Processing status
+ */
+
+#voice-energy-container.processing {
+  #purple-neon-blur {
+    animation: processingBouncePurpleNeonBlur 1s infinite alternate;
+  }
+  #pink-neon-1 {
+    animation: processingBouncePinkNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-1 {
+    animation: processingMoveBlueNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-2 {
+    animation: processingMoveBlueNeon2 0.5s infinite alternate;
+  }
+}
+
+@keyframes processingBouncePurpleNeonBlur {
+  50% {
+    transform: scale(1.07);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes processingBouncePinkNeon1 {
+  50% {
+    transform: scale(1.02);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes processingMoveBlueNeon1 {
+  0% {
+    transform: translateX(0) translateY(0);
+  }
+  33% {
+    transform: translateY(-3px) translateX(-2px);
+  }
+  66% {
+    transform: translateY(-3px) translateX(3px);
+  }
+  100% {
+    transform: translateY(-3px) translateX(1px);
+  }
+}
+@keyframes processingMoveBlueNeon2 {
+  0% {
+    transform: translateX(0) translateY(0);
+  }
+  33% {
+    transform: translateY(3px) translateX(2px);
+  }
+  66% {
+    transform: translateY(3px) translateX(3px);
+  }
+  100% {
+    transform: translateY(3px) translateX(-1px);
+  }
+}
+
+@for $i from 0 through 31 {
+  .processing .voice-particle[data-particle='#{$i}'] {
+    animation-delay: #{$i * 0.1}s;
+  }
+
+  .processing .voice-particle[data-particle='#{$i}'] {
+    animation-name: processingMoveParticle#{$i};
+  }
+
+  #voice-energy-container.processing {
+    @keyframes processingMoveParticle#{$i} {
+      75% {
+        opacity: 0.1;
+      }
+      100% {
+        opacity: 1;
+        transform: translateX(math.cos(11.25deg * $i) * 110px)
+          translateY(math.sin(11.25deg * $i) * 110px);
+      }
+    }
+  }
+}

app/src/css/voice-energy/talking.scss

@@ -0,0 +1,82 @@
+@use 'sass:math';
+
+/**
+ * Talking status
+ */
+
+#voice-energy-container.talking {
+  .voice-neon {
+    margin: 0;
+  }
+  #purple-neon-blur {
+    animation: talkingBouncePurpleNeonBlur 1s infinite alternate;
+  }
+  #pink-neon-1 {
+    transform: scale(1);
+    animation: talkingBouncePinkNeon1 0.5s 1s infinite alternate;
+  }
+  #blue-neon-1 {
+    transform: scale(0.8);
+    animation: talkingMoveBlueNeon1 0.5s infinite alternate;
+  }
+  #blue-neon-2 {
+    transform: scale(0.9);
+    animation: talkingMoveBlueNeon2 0.5s 0.3s infinite alternate;
+  }
+}
+
+@keyframes talkingBouncePurpleNeonBlur {
+  50% {
+    transform: scale(1.07);
+  }
+  100% {
+    transform: scale(1);
+  }
+}
+@keyframes talkingBouncePinkNeon1 {
+  100% {
+    transform: scale(1.1);
+  }
+}
+@keyframes talkingMoveBlueNeon1 {
+  100% {
+    transform: scale(0.9);
+  }
+}
+@keyframes talkingMoveBlueNeon2 {
+  100% {
+    transform: scale(1);
+  }
+}
+
+.talking .voice-particle {
+  opacity: 0;
+  animation-duration: 2s;
+}
+@for $i from 0 through 31 {
+  .talking .voice-particle[data-particle='#{$i}'] {
+    animation-delay: #{$i * 0.2}s;
+    // animation-duration: #{$i * 0.5}s;
+  }
+
+  .talking .voice-particle[data-particle='#{$i}'] {
+    animation-name: talkingMoveParticle#{$i};
+  }
+
+  #voice-energy-container.talking {
+    @keyframes talkingMoveParticle#{$i} {
+      0% {
+        opacity: 1;
+        transform: translate(0);
+      }
+      50% {
+        opacity: 0;
+      }
+      100% {
+        opacity: 0;
+        transform: translateX(math.cos(math.random() * 360deg))
+          translateY(math.sin(math.random() * 360deg));
+      }
+    }
+  }
+}

app/src/custom-aurora-components/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/index.ts

@@ -0,0 +1 @@
+export * from './timer'

app/src/custom-aurora-components/timer/timer.tsx

@@ -0,0 +1,70 @@
+import React, { useState, useEffect } from 'react'
+import { CircularProgress, Flexbox, Text } from '@leon-ai/aurora'
+
+interface TimerProps {
+  initialTime: number
+  interval: number
+  totalTimeContent: string
+  initialProgress?: number
+  onEnd?: () => void
+}
+
+function formatTime(seconds: number): string {
+  const minutes = seconds >= 60 ? Math.floor(seconds / 60) : 0
+  const remainingSeconds = seconds % 60
+  const formattedMinutes = minutes < 10 ? `0${minutes}` : minutes
+  const formattedSeconds =
+    remainingSeconds < 10 ? `0${remainingSeconds}` : remainingSeconds
+
+  return `${formattedMinutes}:${formattedSeconds}`
+}
+
+export function Timer({
+  initialTime,
+  initialProgress,
+  interval,
+  totalTimeContent,
+  onEnd
+}: TimerProps) {
+  const [progress, setProgress] = useState(initialProgress || 0)
+  const [timeLeft, setTimeLeft] = useState(initialTime)
+
+  useEffect(() => {
+    setTimeLeft(initialTime)
+    setProgress(progress)
+  }, [initialTime])
+
+  useEffect(() => {
+    if (timeLeft <= 0) {
+      return
+    }
+
+    const timer = setInterval(() => {
+      setTimeLeft((prevTime) => {
+        const newTime = prevTime - 1
+
+        if (newTime <= 0 && onEnd) {
+          onEnd()
+        }
+
+        return newTime
+      })
+      setProgress((prevProgress) => prevProgress + 100 / initialTime)
+    }, interval)
+
+    return () => clearInterval(timer)
+  }, [initialTime, interval, timeLeft])
+
+  return (
+    <CircularProgress value={progress} size="lg">
+      <Flexbox gap="xs" alignItems="center" justifyContent="center">
+        <Text fontSize="lg" fontWeight="semi-bold">
+          {formatTime(timeLeft)}
+        </Text>
+        <Text fontSize="xs" secondary>
+          {totalTimeContent}
+        </Text>
+      </Flexbox>
+    </CircularProgress>
+  )
+}

app/src/index.html

@@ -1,15 +1,306 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <link rel="stylesheet" href="/css/style.css" />
     <link rel="icon" type="image/png" href="/img/favicon.png" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Leon</title>
+    <style>
+      @import './css/style.scss';
+    </style>
   </head>
   <body class="settingup">
+    <div id="init"></div>
+    <div id="voice-overlay-bg">
+      <div id="voice-container">
+        <div>
+          <div id="voice-energy-container">
+            <svg
+              class="voice-neon"
+              id="pink-neon-1"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_16)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="113.5"
+                  stroke="#FFF"
+                  stroke-width="10"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_16"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="1"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_16"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.929412 0 0 0 0 0.160784 0 0 0 0 0.478431 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_16"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              class="voice-neon"
+              id="blue-neon-1"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_24)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="116.5"
+                  stroke="#FFF"
+                  stroke-width="4"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_24"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="1"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_24"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.109804 0 0 0 0 0.458824 0 0 0 0 0.858824 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_24"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              class="voice-neon"
+              id="blue-neon-2"
+              viewBox="0 0 237 237"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g filter="url(#filter0_i_6_24)">
+                <circle
+                  cx="118.5"
+                  cy="118.5"
+                  r="116.5"
+                  stroke="#FFF"
+                  stroke-width="4"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_i_6_24"
+                  x="0"
+                  y="0"
+                  width="237"
+                  height="237"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feColorMatrix
+                    in="SourceAlpha"
+                    type="matrix"
+                    values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
+                    result="hardAlpha"
+                  />
+                  <feMorphology
+                    radius="5"
+                    operator="erode"
+                    in="SourceAlpha"
+                    result="effect1_innerShadow_6_24"
+                  />
+                  <feOffset />
+                  <feGaussianBlur stdDeviation="1" />
+                  <feComposite
+                    in2="hardAlpha"
+                    operator="arithmetic"
+                    k2="-1"
+                    k3="1"
+                  />
+                  <feColorMatrix
+                    type="matrix"
+                    values="0 0 0 0 0.109804 0 0 0 0 0.458824 0 0 0 0 0.858824 0 0 0 1 0"
+                  />
+                  <feBlend
+                    mode="normal"
+                    in2="shape"
+                    result="effect1_innerShadow_6_24"
+                  />
+                </filter>
+              </defs>
+            </svg>
+
+            <svg
+              id="purple-neon-blur"
+              viewBox="0 0 341 341"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <g opacity="0.72" filter="url(#filter0_f_18_61)">
+                <circle
+                  cx="170.5"
+                  cy="170.5"
+                  r="116.5"
+                  stroke="#ED297A"
+                  stroke-width="24"
+                />
+              </g>
+              <g opacity="0.72" filter="url(#filter1_f_18_61)">
+                <circle
+                  cx="170.5"
+                  cy="170.5"
+                  r="116.5"
+                  stroke="#1C75DB"
+                  stroke-width="24"
+                />
+              </g>
+              <defs>
+                <filter
+                  id="filter0_f_18_61"
+                  x="0"
+                  y="0"
+                  width="341"
+                  height="341"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feGaussianBlur
+                    stdDeviation="21"
+                    result="effect1_foregroundBlur_18_61"
+                  />
+                </filter>
+                <filter
+                  id="filter1_f_18_61"
+                  x="0"
+                  y="0"
+                  width="341"
+                  height="341"
+                  filterUnits="userSpaceOnUse"
+                  color-interpolation-filters="sRGB"
+                >
+                  <feFlood flood-opacity="0" result="BackgroundImageFix" />
+                  <feBlend
+                    mode="normal"
+                    in="SourceGraphic"
+                    in2="BackgroundImageFix"
+                    result="shape"
+                  />
+                  <feGaussianBlur
+                    stdDeviation="21"
+                    result="effect1_foregroundBlur_18_61"
+                  />
+                </filter>
+              </defs>
+            </svg>
+          </div>
+          <div id="voice-status"></div>
+          <div id="voice-tips">
+            It is recommended to use a headset for a better voice experience.
+            <br />
+            Otherwise, if your microphone is too sensitive or speakers are too
+            loud,
+            <br />
+            Leon may hear his own voice and get confused.
+          </div>
+        </div>
+        <p id="voice-speech"></p>
+      </div>
+    </div>
     <main>
-      <div id="root"></div>
+      <div id="top-container">
+        <div id="mood"></div>
+        <button id="info">Info</button>
+      </div>
       <div id="feed">
         <p id="no-bubble" class="hide">
           You can start to interact with Leon, don't be shy.
@@ -27,16 +318,20 @@
           <div id="sonar"></div>
         </div>
         <label for="utterance"></label>
-        <input type="text" id="utterance" autocomplete="off" autofocus />
-        <small>
-          Use <kbd>↑</kbd> <kbd>↓</kbd> to browse history; <kbd>↵</kbd> to
-          submit;
-          <kbd>alt + c to listen.</kbd>
-        </small>
+        <textarea id="utterance" autocomplete="off" autofocus></textarea>
+        <ul id="tip">
+          <li><kbd>enter</kbd> to submit.</li>
+          <li><kbd>shift</kbd> + <kbd>enter</kbd> for new line.</li>
+          <li>
+            <kbd>shift</kbd> + <kbd>↑</kbd> / <kbd>↓</kbd> to browse history.
+          </li>
+          <li><kbd>alt</kbd> / <kbd>cmd</kbd> + <kbd>c</kbd> to listen.</li>
+        </ul>
       </div>
     </main>
     <footer>
       <div id="logo"></div>
+      <br />
       <div id="version">
         <small>v</small>
       </div>

app/src/js/app.tsx

@@ -1,9 +0,0 @@
-function App() {
-  return (
-    <>
-      <p>Hello from React</p>
-    </>
-  )
-}
-
-export default App

app/src/js/aurora/button.tsx

@@ -1,9 +0,0 @@
-import type React from 'react'
-
-interface Props {
-  children: React.ReactNode
-}
-
-export function Button({ children }: Props) {
-  return <button>{children}</button>
-}

app/src/js/chatbot.js

@@ -1,7 +1,17 @@
-const MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE = 340
+import { createElement } from 'react'
+import { createRoot } from 'react-dom/client'
+import axios from 'axios'
+import { WidgetWrapper, Flexbox, Loader, Text } from '@leon-ai/aurora'
+
+import renderAuroraComponent from './render-aurora-component'
+
+const WIDGETS_TO_FETCH = []
+const WIDGETS_FETCH_CACHE = new Map()
 
 export default class Chatbot {
-  constructor() {
+  constructor(socket, serverURL) {
+    this.socket = socket
+    this.serverURL = serverURL
     this.et = new EventTarget()
     this.feed = document.querySelector('#feed')
     this.typing = document.querySelector('#is-typing')
@@ -15,11 +25,17 @@ export default class Chatbot {
     this.scrollDown()
 
     this.et.addEventListener('to-leon', (event) => {
-      this.createBubble('me', event.detail)
+      this.createBubble({
+        who: 'me',
+        string: event.detail
+      })
     })
 
     this.et.addEventListener('me-received', (event) => {
-      this.createBubble('leon', event.detail)
+      this.createBubble({
+        who: 'leon',
+        string: event.detail
+      })
     })
   }
 
@@ -62,10 +78,7 @@ export default class Chatbot {
   }
 
   loadFeed() {
-    /**
-     * TODO: widget: load widget from local storage
-     */
-    return new Promise((resolve) => {
+    return new Promise(async (resolve) => {
       if (this.parsedBubbles === null || this.parsedBubbles.length === 0) {
         this.noBubbleMessage.classList.remove('hide')
         localStorage.setItem('bubbles', JSON.stringify([]))
@@ -75,7 +88,12 @@ export default class Chatbot {
         for (let i = 0; i < this.parsedBubbles.length; i += 1) {
           const bubble = this.parsedBubbles[i]
 
-          this.createBubble(bubble.who, bubble.string, false)
+          this.createBubble({
+            who: bubble.who,
+            string: bubble.string,
+            save: false,
+            isCreatingFromLoadingFeed: true
+          })
 
           if (i + 1 === this.parsedBubbles.length) {
             setTimeout(() => {
@@ -83,40 +101,150 @@ export default class Chatbot {
             }, 100)
           }
         }
+
+        /**
+         * Browse widgets that need to be fetched.
+         * Reverse widgets to fetch the last widgets first.
+         * Replace the loading content with the fetched widget
+         */
+        const widgetContainers = WIDGETS_TO_FETCH.reverse()
+        for (let i = 0; i < widgetContainers.length; i += 1) {
+          const widgetContainer = widgetContainers[i]
+          const hasWidgetBeenFetched = WIDGETS_FETCH_CACHE.has(
+            widgetContainer.widgetId
+          )
+
+          if (hasWidgetBeenFetched) {
+            const fetchedWidget = WIDGETS_FETCH_CACHE.get(
+              widgetContainer.widgetId
+            )
+            widgetContainer.reactRootNode.render(fetchedWidget.reactNode)
+
+            setTimeout(() => {
+              this.scrollDown()
+            }, 100)
+
+            continue
+          }
+
+          const data = await axios.get(
+            `${this.serverURL}/api/v1/fetch-widget?skill_action=${widgetContainer.onFetch.actionName}&widget_id=${widgetContainer.widgetId}`
+          )
+          const fetchedWidget = data.data.widget
+          const reactNode = fetchedWidget
+            ? renderAuroraComponent(
+                this.socket,
+                fetchedWidget.componentTree,
+                fetchedWidget.supportedEvents
+              )
+            : createElement(WidgetWrapper, {
+                children: createElement(Flexbox, {
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  children: createElement(Text, {
+                    secondary: true,
+                    children: 'This widget has been deleted.'
+                  })
+                })
+              })
+
+          widgetContainer.reactRootNode.render(reactNode)
+          WIDGETS_FETCH_CACHE.set(widgetContainer.widgetId, {
+            ...fetchedWidget,
+            reactNode
+          })
+          setTimeout(() => {
+            this.scrollDown()
+          }, 100)
+        }
       }
     })
   }
 
-  createBubble(who, string, save = true) {
+  createBubble(params) {
+    const {
+      who,
+      string,
+      save = true,
+      bubbleId,
+      isCreatingFromLoadingFeed = false
+    } = params
     const container = document.createElement('div')
     const bubble = document.createElement('p')
 
     container.className = `bubble-container ${who}`
     bubble.className = 'bubble'
-    bubble.innerHTML = string
+
+    const formattedString = this.formatMessage(string)
+
+    bubble.innerHTML = formattedString
+
+    if (bubbleId) {
+      container.classList.add(bubbleId)
+    }
 
     this.feed.appendChild(container).appendChild(bubble)
 
-    if (container.clientHeight > MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE) {
-      bubble.style.maxHeight = `${MAXIMUM_HEIGHT_TO_SHOW_SEE_MORE}px`
-      const showMore = document.createElement('p')
-      const showMoreText = 'Show more'
+    let widgetComponentTree = null
+    let widgetSupportedEvents = null
 
-      showMore.className = 'show-more'
-      showMore.innerHTML = showMoreText
+    /**
+     * Widget rendering
+     */
+    if (
+      formattedString.includes &&
+      formattedString.includes('"component":"WidgetWrapper"')
+    ) {
+      const parsedWidget = JSON.parse(formattedString)
+      container.setAttribute('data-widget-id', parsedWidget.id)
 
-      container.appendChild(showMore)
+      /**
+       * On widget fetching, render the loader
+       */
+      if (isCreatingFromLoadingFeed && parsedWidget.onFetch) {
+        const root = createRoot(container)
 
-      showMore.addEventListener('click', () => {
-        bubble.classList.toggle('show-all')
-        showMore.innerHTML =
-          showMore.innerHTML === showMoreText ? 'Show less' : showMoreText
-      })
+        root.render(
+          createElement(WidgetWrapper, {
+            children: createElement(Flexbox, {
+              alignItems: 'center',
+              justifyContent: 'center',
+              children: createElement(Loader)
+            })
+          })
+        )
+
+        WIDGETS_TO_FETCH.push({
+          reactRootNode: root,
+          widgetId: parsedWidget.id,
+          onFetch: parsedWidget.onFetch
+        })
+
+        return
+      }
+
+      widgetComponentTree = parsedWidget.componentTree
+      widgetSupportedEvents = parsedWidget.supportedEvents
+
+      /**
+       * On widget creation
+       */
+      const root = createRoot(container)
+
+      const reactNode = renderAuroraComponent(
+        this.socket,
+        widgetComponentTree,
+        widgetSupportedEvents
+      )
+
+      root.render(reactNode)
     }
 
     if (save) {
-      this.saveBubble(who, string)
+      this.saveBubble(who, formattedString)
     }
+
+    return container
   }
 
   saveBubble(who, string) {
@@ -132,4 +260,12 @@ export default class Chatbot {
     localStorage.setItem('bubbles', JSON.stringify(this.parsedBubbles))
     this.scrollDown()
   }
+
+  formatMessage(message) {
+    if (typeof message === 'string') {
+      message = message.replace(/\n/g, '<br />')
+    }
+
+    return message
+  }
 }

app/src/js/client.js

@@ -1,24 +1,28 @@
 import { io } from 'socket.io-client'
-import React from 'react'
-import { createRoot } from 'react-dom/client'
 
-import { Button } from './aurora/button'
 import Chatbot from './chatbot'
+import VoiceEnergy from './voice-energy'
 import { INIT_MESSAGES } from './constants'
 
 export default class Client {
-  constructor(client, serverUrl, input, res) {
+  constructor(client, serverUrl, input) {
     this.client = client
     this._input = input
     this._suggestionContainer = document.querySelector('#suggestions-container')
+    this.voiceSpeechElement = document.querySelector('#voice-speech')
     this.serverUrl = serverUrl
     this.socket = io(this.serverUrl)
     this.history = localStorage.getItem('history')
     this.parsedHistory = []
-    this.info = res
-    this.chatbot = new Chatbot()
+    this.chatbot = new Chatbot(this.socket, this.serverUrl)
+    this.voiceEnergy = new VoiceEnergy(this)
     this._recorder = {}
     this._suggestions = []
+    this._answerGenerationId = 'xxx'
+    this._ttsAudioContext = null
+    this._isLeonGeneratingAnswer = false
+    this._isVoiceModeEnabled = false
+    // this._ttsAudioContextes = {}
   }
 
   set input(newInput) {
@@ -35,6 +39,15 @@ export default class Client {
     return this._recorder
   }
 
+  updateMood(mood) {
+    if (window.leonConfigInfo.llm.enabled) {
+      const moodContainer = document.querySelector('#mood')
+
+      moodContainer.textContent = `Leon's mood: ${mood.emoji}`
+      moodContainer.setAttribute('title', mood.type)
+    }
+  }
+
   async sendInitMessages() {
     for (let i = 0; i < INIT_MESSAGES.length; i++) {
       const messages = INIT_MESSAGES[i]
@@ -53,15 +66,61 @@ export default class Client {
     }
   }
 
-  init(loader) {
+  setInitStatus(statusName, statusType) {
+    window.leonInitStatusEvent.dispatchEvent(
+      new CustomEvent('initStatusChange', {
+        detail: {
+          statusName,
+          statusType
+        }
+      })
+    )
+  }
+
+  asrStartRecording() {
+    if (!window.leonConfigInfo.stt.enabled) {
+      console.warn('ASR is not enabled')
+      return
+    }
+
+    if (!this._isVoiceModeEnabled) {
+      this.enableVoiceMode()
+
+      this.voiceEnergy.status = 'listening'
+
+      this.socket.emit('asr-start-record')
+    }
+  }
+
+  init() {
     this.chatbot.init()
+    this.voiceEnergy.init()
 
     this.socket.on('connect', () => {
       this.socket.emit('init', this.client)
     })
 
+    /**
+     * Init status listeners
+     */
+    this.socket.on('init-client-core-server-handshake', (status) => {
+      this.setInitStatus('clientCoreServerHandshake', status)
+    })
+    this.socket.on('init-tcp-server-boot', (status) => {
+      this.setInitStatus('tcpServerBoot', status)
+    })
+    this.socket.on('init-llm', (status) => {
+      this.setInitStatus('llm', status)
+    })
+    this.socket.on('warmup-llm-duties', (status) => {
+      this.setInitStatus('llmDutiesWarmUp', status)
+    })
+
     this.socket.on('ready', () => {
-      loader.stop()
+      setTimeout(() => {
+        const body = document.querySelector('body')
+        body.classList.remove('settingup')
+      }, 250)
 
       if (this.chatbot.parsedBubbles?.length === 0) {
         this.sendInitMessages()
@@ -69,7 +128,36 @@ export default class Client {
     })
 
     this.socket.on('answer', (data) => {
-      this.chatbot.receivedFrom('leon', data)
+      /*if (this._isVoiceModeEnabled) {
+        this.voiceEnergy.status = 'listening'
+      }*/
+
+      // Leon has finished to answer
+      this._isLeonGeneratingAnswer = false
+
+      /**
+       * Just save the bubble if the newest bubble is from the streaming.
+       * Otherwise, create a new bubble
+       */
+      const newestBubbleContainerElement =
+        document.querySelector('.leon:last-child')
+      const isNewestBubbleFromStreaming =
+        newestBubbleContainerElement?.classList.contains(
+          this._answerGenerationId
+        )
+
+      if (isNewestBubbleFromStreaming) {
+        this.chatbot.saveBubble('leon', data)
+
+        // Slightly delay the update to avoid the stream animation to be interrupted
+        setTimeout(() => {
+          // Update the text of the bubble (quick emoji fix)
+          newestBubbleContainerElement.querySelector('p.bubble').innerHTML =
+            this.chatbot.formatMessage(data)
+        }, 2_500)
+      } else {
+        this.chatbot.receivedFrom('leon', data)
+      }
     })
 
     this.socket.on('suggest', (data) => {
@@ -90,27 +178,121 @@ export default class Client {
     })
 
     this.socket.on('widget', (data) => {
-      /**
-       * TODO: widget: widget handler to core/skill; dynamic component rendering
-       */
-      console.log('data', data)
+      this.chatbot.createBubble({
+        who: 'leon',
+        string: data
+      })
+    })
 
-      const container = document.createElement('div')
-      container.className = 'widget'
-      this.chatbot.feed.appendChild(container)
+    this.socket.on('widget-send-utterance', (utterance) => {
+      this._input.value = utterance
+      this.send('utterance')
+    })
 
-      const root = createRoot(container)
+    this.socket.on('new-mood', (mood) => {
+      this.updateMood(mood)
+    })
 
-      // TODO: widget: pass props and dynamic component loading according to type
-      const widgets = {
-        Button: (options) => {
-          return Button({
-            children: options.text
-          })
-        }
+    this.socket.on('llm-token', (data) => {
+      if (this._isVoiceModeEnabled) {
+        this.voiceEnergy.status = 'processing'
       }
 
-      root.render(widgets[data.type](data.options))
+      this._isLeonGeneratingAnswer = true
+      const previousGenerationId = this._answerGenerationId
+      const newGenerationId = data.generationId
+      this._answerGenerationId = newGenerationId
+      const isSameGeneration = previousGenerationId === newGenerationId
+      let bubbleContainerElement = null
+
+      if (!isSameGeneration) {
+        bubbleContainerElement = this.chatbot.createBubble({
+          who: 'leon',
+          string: data.token,
+          save: false,
+          bubbleId: newGenerationId
+        })
+      } else {
+        bubbleContainerElement = document.querySelector(
+          `.${previousGenerationId}`
+        )
+      }
+
+      const bubbleElement = bubbleContainerElement.querySelector('p.bubble')
+
+      // Token is already appened when it's a new generation
+      if (isSameGeneration) {
+        // bubbleElement.textContent += data.token
+
+        const tokenSpan = document.createElement('span')
+        tokenSpan.className = 'llm-token fade-in'
+        tokenSpan.textContent = data.token
+
+        bubbleElement.appendChild(tokenSpan)
+      }
+
+      this.chatbot.scrollDown()
+    })
+
+    this.socket.on('asr-speech', (text) => {
+      if (!this._isVoiceModeEnabled) {
+        this.enableVoiceMode()
+      }
+
+      this.voiceEnergy.status = 'listening'
+      this._input.value = text
+
+      if (this.voiceSpeechElement) {
+        this.voiceSpeechElement.textContent = text
+      }
+    })
+
+    this.socket.on('asr-end-of-owner-speech', () => {
+      this.voiceEnergy.status = 'processing'
+
+      setTimeout(() => {
+        this.send('utterance')
+      }, 200)
+    })
+
+    this.socket.on('asr-active-listening-disabled', () => {
+      this.voiceEnergy.status = 'idle'
+    })
+
+    /**
+     * Only used for "local" TTS provider as a PoC for now.
+     * Target to do a better implementation in the future
+     * with streaming support
+     */
+    this.socket.on('tts-stream', (data) => {
+      this.voiceEnergy.status = 'talking'
+
+      // const { audioId, chunk } = data
+      const { chunk } = data
+      this._ttsAudioContext = new AudioContext()
+      // this._ttsAudioContextes[audioId] = ctx
+
+      const source = this._ttsAudioContext.createBufferSource()
+      this._ttsAudioContext.decodeAudioData(chunk, (buffer) => {
+        source.buffer = buffer
+
+        source.connect(this._ttsAudioContext.destination)
+        source.start(0)
+      })
+    })
+
+    /**
+     * When Leon got interrupted by the owner voice
+     * while he is speaking
+     */
+    this.socket.on('tts-interruption', async () => {
+      if (this._ttsAudioContext) {
+        await this._ttsAudioContext.close()
+      }
+    })
+
+    this.socket.on('tts-end-of-speech', async () => {
+      this.voiceEnergy.status = 'listening'
     })
 
     this.socket.on('audio-forwarded', (data, cb) => {
@@ -127,7 +309,7 @@ export default class Client {
          * When the after speech option is enabled and
          * the answer is a final one
          */
-        if (this.info.after_speech && data.is_final_answer) {
+        if (window.leonConfigInfo.after_speech && data.is_final_answer) {
           // Enable recording after the speech + 500ms
           setTimeout(() => {
             this._recorder.start()
@@ -164,6 +346,11 @@ export default class Client {
   }
 
   send(keyword) {
+    // Prevent from sending utterance if Leon is still generating text (stream)
+    if (keyword === 'utterance' && this._isLeonGeneratingAnswer) {
+      return false
+    }
+
     if (this._input.value !== '') {
       this.socket.emit(keyword, {
         client: this.client,
@@ -205,6 +392,10 @@ export default class Client {
     }
 
     this._input.value = ''
+    setTimeout(() => {
+      // Remove the last character to avoid the space
+      this._input.value = this._input.value.slice(0, -1)
+    }, 0)
   }
 
   addSuggestion(text) {
@@ -222,4 +413,41 @@ export default class Client {
 
     this._suggestions.push(newSuggestion)
   }
+
+  enableVoiceMode() {
+    if (!this._isVoiceModeEnabled) {
+      this._isVoiceModeEnabled = true
+
+      const body = document.querySelector('body')
+      if (!body.classList.contains('voice-mode-enabled')) {
+        body.classList.add('voice-mode-enabled')
+
+        const voiceOverlayTransitor = document.createElement('div')
+        voiceOverlayTransitor.id = 'voice-overlay-transitor'
+        body.appendChild(voiceOverlayTransitor)
+        voiceOverlayTransitor.addEventListener('animationend', () => {
+          voiceOverlayTransitor.removeEventListener('animationend', () => {})
+          voiceOverlayTransitor.remove()
+        })
+      }
+    }
+  }
+  disableVoiceMode() {
+    if (this._isVoiceModeEnabled) {
+      this._isVoiceModeEnabled = false
+
+      const body = document.querySelector('body')
+
+      const voiceContainer = document.querySelector('#voice-container')
+      if (voiceContainer) {
+        voiceContainer.style.animation = 'none'
+        voiceContainer.offsetHeight
+        voiceContainer.style.animation = null
+      }
+
+      if (body.classList.contains('voice-mode-enabled')) {
+        body.classList.remove('voice-mode-enabled')
+      }
+    }
+  }
 }

app/src/js/constants.js

@@ -12,9 +12,6 @@ export const INIT_MESSAGES = [
   [
     `Come hang out with us <a href="https://discord.gg/MNQqqKg" target="_blank">on Discord</a>! Once we release our official version, our community will be working together to build new skills for me. You won't want to miss out on the fun!`
   ],
-  [
-    `At the moment, I'm not using a large language model, but we're planning to incorporate some in the future to improve my abilities. You can <a href="https://github.com/leon-ai/leon#how-about-large-language-models-and-leon" target="_blank">learn more about our plans here</a>.`
-  ],
   [
     `Just so you know, my creator is working tirelessly to improve my skills and features, dedicating 75% of his free time to the project on top of his full-time job. If you'd like to help speed up my development, you can sponsor his work by clicking on this link: <strong><a href='http://sponsor.getleon.ai/' target='_blank'>sponsor.getleon.ai</a></strong>. Your support would mean a lot to us. Thank you for choosing me as your assistant!`
   ]

app/src/js/init.jsx

@@ -0,0 +1,193 @@
+import { useEffect, useState, useRef } from 'react'
+import { createRoot } from 'react-dom/client'
+import {
+  WidgetWrapper,
+  Text,
+  Icon,
+  Flexbox,
+  List,
+  ListHeader,
+  ListItem,
+  Loader
+} from '@leon-ai/aurora'
+
+const container = document.querySelector('#init')
+const root = createRoot(container)
+
+function Item({ children, status }) {
+  if (status === 'error') {
+    return <ErrorListItem>{children}</ErrorListItem>
+  }
+  if (status === 'warning') {
+    return <WarningListItem>{children}</WarningListItem>
+  }
+  if (status === 'success') {
+    return <SuccessListItem>{children}</SuccessListItem>
+  }
+  if (status === 'loading') {
+    return <LoadingListItem>{children}</LoadingListItem>
+  }
+
+  return <ListItem>{children}</ListItem>
+}
+
+function LoadingListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Loader size="sm" />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function ErrorListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="close"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="red"
+          bgColor="transparent-red"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function WarningListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="alert"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="yellow"
+          bgColor="transparent-yellow"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+function SuccessListItem({ children }) {
+  return (
+    <ListItem>
+      <Flexbox flexDirection="row" alignItems="center" gap="sm">
+        <Icon
+          iconName="check"
+          size="sm"
+          type="fill"
+          bgShape="circle"
+          color="green"
+          bgColor="transparent-green"
+        />
+        <Text>{children}</Text>
+      </Flexbox>
+    </ListItem>
+  )
+}
+
+function Init() {
+  const parentRef = useRef(null)
+  const [config, setConfig] = useState(() => ({ ...window.leonConfigInfo }))
+  const [statusMap, setStatusMap] = useState({
+    clientCoreServerHandshake: 'loading',
+    tcpServerBoot: 'loading',
+    llm: 'loading',
+    llmDutiesWarmUp: 'loading'
+  })
+
+  useEffect(() => {
+    setTimeout(() => {
+      if (parentRef.current) {
+        parentRef.current.classList.remove('not-initialized')
+      }
+    }, 250)
+
+    function handleStatusChange(event) {
+      const { statusName, statusType } = event.detail
+
+      setStatusMap((prev) => ({ ...prev, [statusName]: statusType }))
+    }
+
+    window.leonInitStatusEvent.addEventListener(
+      'initStatusChange',
+      handleStatusChange
+    )
+    return () =>
+      window.leonInitStatusEvent.removeEventListener(
+        'initStatusChange',
+        handleStatusChange
+      )
+  }, [])
+
+  const statuses = []
+  for (let key of Object.keys(statusMap)) {
+    // If LLM is not enabled, we don't need to check for LLM duties warm up
+    if (
+      key === 'llmDutiesWarmUp' &&
+      (!config.llm?.enabled || !config.shouldWarmUpLLMDuties)
+    ) {
+      statuses.push('success')
+    } else if (!config[key] || config[key].enabled) {
+      statuses.push(statusMap[key])
+    }
+  }
+
+  const areAllStatusesSuccess = statuses.every((status) => status === 'success')
+
+  useEffect(() => {
+    if (window.leonConfigInfo) {
+      setConfig({ ...window.leonConfigInfo })
+    }
+  }, [window.leonConfigInfo])
+
+  return (
+    <div
+      style={{
+        position: 'fixed',
+        width: '100vw',
+        height: '100vh',
+        zIndex: 9999,
+        backgroundColor: 'var(--black-color)'
+      }}
+      ref={parentRef}
+      className={areAllStatusesSuccess ? 'initialized' : 'not-initialized'}
+    >
+      <div
+        style={{
+          position: 'absolute',
+          top: '33%',
+          left: '50%',
+          transform: 'translate(-50%, -50%)'
+        }}
+      >
+        <WidgetWrapper noPadding>
+          <List>
+            <ListHeader>Leon is getting ready...</ListHeader>
+            <Item status={statusMap.clientCoreServerHandshake}>
+              Client and core server handshaked
+            </Item>
+            <Item status={statusMap.tcpServerBoot}>TCP server booted</Item>
+            {config.llm && config.llm.enabled && (
+              <Item status={statusMap.llm}>LLM loaded</Item>
+            )}
+            {config.shouldWarmUpLLMDuties && (
+              <Item status={statusMap.llmDutiesWarmUp}>
+                LLM duties warmed up
+              </Item>
+            )}
+          </List>
+        </WidgetWrapper>
+      </div>
+    </div>
+  )
+}
+
+root.render(<Init />)

app/src/js/loader.js

@@ -1,22 +0,0 @@
-export default class Loader {
-  constructor() {
-    this.et = new EventTarget()
-    this.body = document.querySelector('body')
-
-    this.et.addEventListener('settingup', (event) => {
-      if (event.detail) {
-        this.body.classList.add('settingup')
-      } else {
-        this.body.classList.remove('settingup')
-      }
-    })
-  }
-
-  start() {
-    this.et.dispatchEvent(new CustomEvent('settingup', { detail: true }))
-  }
-
-  stop() {
-    this.et.dispatchEvent(new CustomEvent('settingup', { detail: false }))
-  }
-}

app/src/js/main.js

@@ -1,10 +1,13 @@
 import axios from 'axios'
+import '@leon-ai/aurora/style.css'
 
-import Loader from './loader'
+window.leonInitStatusEvent = new EventTarget()
+
+import './init'
 import Client from './client'
-import Recorder from './recorder'
-import listener from './listener'
-import { onkeydowndocument, onkeydowninput } from './onkeydown'
+// import Recorder from './recorder'
+// import listener from './listener'
+import { onkeydownstartrecording, onkeydowninput } from './onkeydown'
 
 const config = {
   app: 'webapp',
@@ -19,29 +22,54 @@ const serverUrl =
     : `${config.server_host}:${config.server_port}`
 
 document.addEventListener('DOMContentLoaded', async () => {
-  const loader = new Loader()
-
-  loader.start()
-
   try {
     const response = await axios.get(`${serverUrl}/api/v1/info`)
     const input = document.querySelector('#utterance')
     const mic = document.querySelector('#mic-button')
     const v = document.querySelector('#version small')
-    const client = new Client(config.app, serverUrl, input, response.data)
-    let rec = {}
-    let chunks = []
+    const infoButton = document.querySelector('#info')
+    const client = new Client(config.app, serverUrl, input)
+    // let rec = {}
+    // let chunks = []
 
-    v.innerHTML += client.info.version
+    window.leonConfigInfo = response.data
+    const infoKeys = [
+      'timeZone',
+      'telemetry',
+      'gpu',
+      'graphicsComputeAPI',
+      'totalVRAM',
+      'freeVRAM',
+      'usedVRAM',
+      'llm',
+      'shouldWarmUpLLMDuties',
+      'isLLMActionRecognitionEnabled',
+      'isLLMNLGEnabled',
+      'stt',
+      'tts',
+      'mood',
+      'version'
+    ]
+    const infoToDisplay = {}
+    infoKeys.forEach((key) => {
+      infoToDisplay[key] = window.leonConfigInfo[key]
+    })
 
-    client.init(loader)
+    v.textContent += window.leonConfigInfo.version
 
-    if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+    client.updateMood(window.leonConfigInfo.mood)
+    client.init()
+
+    infoButton.addEventListener('click', () => {
+      alert(JSON.stringify(infoToDisplay, null, 2))
+    })
+
+    /*if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
       navigator.mediaDevices
         .getUserMedia({ audio: true })
         .then((stream) => {
           if (MediaRecorder) {
-            rec = new Recorder(stream, mic, client.info)
+            rec = new Recorder(stream, mic, window.leonConfigInfo)
             client.recorder = rec
 
             rec.ondataavailable((e) => {
@@ -49,7 +77,7 @@ document.addEventListener('DOMContentLoaded', async () => {
             })
 
             rec.onstart(() => {
-              /* */
+              /!* *!/
             })
 
             rec.onstop(() => {
@@ -106,18 +134,19 @@ document.addEventListener('DOMContentLoaded', async () => {
       console.error(
         'MediaDevices.getUserMedia() is not supported on your browser.'
       )
-    }
+    }*/
 
     document.addEventListener('keydown', (e) => {
-      onkeydowndocument(e, () => {
-        if (rec.enabled === false) {
+      onkeydownstartrecording(e, () => {
+        client.asrStartRecording()
+        /*if (rec.enabled === false) {
           input.value = ''
           rec.start()
           rec.enabled = true
         } else {
           rec.stop()
           rec.enabled = false
-        }
+        }*/
       })
     })
 
@@ -128,13 +157,15 @@ document.addEventListener('DOMContentLoaded', async () => {
     mic.addEventListener('click', (e) => {
       e.preventDefault()
 
-      if (rec.enabled === false) {
+      client.asrStartRecording()
+
+      /*if (rec.enabled === false) {
         rec.start()
         rec.enabled = true
       } else {
         rec.stop()
         rec.enabled = false
-      }
+      }*/
     })
   } catch (e) {
     alert(`Error: ${e.message}; ${JSON.stringify(e.response?.data)}`)

app/src/js/onkeydown.js

@@ -8,29 +8,31 @@ const onkeydowninput = (e, client) => {
     parsedHistory = JSON.parse(localStorage.getItem('history')).reverse()
   }
 
-  if (key === 13) {
+  if (key === 13 && !e.shiftKey) {
     if (client.send('utterance')) {
       parsedHistory = JSON.parse(localStorage.getItem('history')).reverse()
       index = -1
     }
   } else if (localStorage.getItem('history') !== null) {
-    if (key === 38 && index < parsedHistory.length - 1) {
-      index += 1
-      client.input = parsedHistory[index]
-    } else if (key === 40 && index - 1 >= 0) {
-      index -= 1
-      client.input = parsedHistory[index]
-    } else if (key === 40 && index - 1 < 0) {
-      client.input = ''
-      index = -1
+    if (e.shiftKey) {
+      if (key === 38 && index < parsedHistory.length - 1) {
+        index += 1
+        client.input = parsedHistory[index]
+      } else if (key === 40 && index - 1 >= 0) {
+        index -= 1
+        client.input = parsedHistory[index]
+      } else if (key === 40 && index - 1 < 0) {
+        client.input = ''
+        index = -1
+      }
     }
   }
 }
 
-const onkeydowndocument = (e, cb) => {
-  if (e.altKey && e.key === 'c') {
+const onkeydownstartrecording = (e, cb) => {
+  if ((e.metaKey || e.altKey) && e.key === 'c') {
     cb()
   }
 }
 
-export { onkeydowninput, onkeydowndocument }
+export { onkeydowninput, onkeydownstartrecording }

app/src/js/render-aurora-component.js

@@ -0,0 +1,53 @@
+import { createElement } from 'react'
+import * as auroraComponents from '@leon-ai/aurora'
+
+import * as customAuroraComponents from '../custom-aurora-components'
+
+export default function renderAuroraComponent(
+  socket,
+  component,
+  supportedEvents
+) {
+  if (component) {
+    // eslint-disable-next-line import/namespace
+    let reactComponent = auroraComponents[component.component]
+    /**
+     * Find custom component if a former component is not found
+     */
+    if (!reactComponent) {
+      // eslint-disable-next-line import/namespace
+      reactComponent = customAuroraComponents[component.component]
+    }
+
+    if (!reactComponent) {
+      console.error(`Component ${component} not found`)
+    }
+
+    // Check if the browsed component has a supported event and bind it
+    if (reactComponent) {
+      component.events.forEach((event) => {
+        if (supportedEvents.includes(event.type)) {
+          component.props[event.type] = (data) => {
+            const { method } = event
+
+            socket.emit('widget-event', { method, data })
+          }
+        }
+      })
+    }
+
+    // When children is a component, then wrap it in an array to render properly
+    const isComponent = !!component.props?.children?.component
+    if (isComponent) {
+      component.props.children = [component.props.children]
+    }
+
+    if (component.props?.children && Array.isArray(component.props.children)) {
+      component.props.children = component.props.children.map((child) => {
+        return renderAuroraComponent(socket, child, supportedEvents)
+      })
+    }
+
+    return createElement(reactComponent, component.props)
+  }
+}

app/src/js/voice-energy.js

@@ -0,0 +1,74 @@
+const STATUS = {
+  listening: 'Listening...',
+  processing: 'Processing...',
+  talking: 'Talking...',
+  idle: 'Idle'
+}
+
+export default class VoiceEnergy {
+  constructor(client) {
+    this.client = client
+    this.voiceEnergyContainerElement = document.querySelector(
+      '#voice-energy-container'
+    )
+    this.voiceOverlayElement = document.querySelector('#voice-overlay-bg')
+    this.statusElement = document.querySelector('#voice-status')
+    this._status = 'idle'
+  }
+
+  get status() {
+    return this._status
+  }
+
+  set status(newStatus) {
+    if (this._status !== newStatus) {
+      this._status = newStatus
+
+      if (this.statusElement) {
+        this.statusElement.textContent = STATUS[newStatus]
+      }
+
+      // Clean up speech text when listening
+      if (newStatus === 'listening' && this.client.voiceSpeechElement) {
+        this.client.voiceSpeechElement.textContent = ''
+      }
+
+      if (this.voiceEnergyContainerElement) {
+        this.voiceEnergyContainerElement.className = ''
+        this.voiceEnergyContainerElement.classList.add(newStatus)
+      }
+    }
+  }
+
+  init() {
+    if (this.voiceEnergyContainerElement) {
+      if (this.voiceOverlayElement) {
+        this.voiceOverlayElement.addEventListener('click', (e) => {
+          e.preventDefault()
+          this.client.disableVoiceMode()
+        })
+      }
+
+      const particles = new Set()
+      const particleColors = ['blue', 'pink']
+
+      for (let i = 0; i < 32; i += 1) {
+        const particle = document.createElement('div')
+        const randomColor = Math.floor(Math.random() * 2)
+        let random = Math.floor(Math.random() * 32)
+
+        while (particles.has(random)) {
+          random = Math.floor(Math.random() * 32)
+        }
+
+        particles.add(random)
+        particle.setAttribute('data-particle', String(random))
+        particle.classList.add('voice-particle', particleColors[randomColor])
+        particle.style.transform = `rotate(${
+          i * 11.25
+        }deg) translate(110px) rotate(-${i * 11.25}deg)`
+        this.voiceEnergyContainerElement.appendChild(particle)
+      }
+    }
+  }
+}

app/tsconfig.json

@@ -1,20 +0,0 @@
-{
-  "compilerOptions": {
-    "useDefineForClassFields": true,
-    "skipLibCheck": true,
-    "module": "ESNext",
-    /* Bundler mode */
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "noEmit": true,
-    "jsx": "react-jsx",
-
-    /* Linting */
-    "strict": true,
-    "noUnusedLocals": true,
-    "noUnusedParameters": true,
-    "noFallthroughCasesInSwitch": true
-  }
-}

bridges/nodejs/src/constants.ts

@@ -1,8 +1,7 @@
 import fs from 'node:fs'
 import path from 'node:path'
 
-import { SKILLS_PATH as SKILLS_ROOT_PATH } from '@/constants'
-import type { SkillConfigSchema } from '@server/schemas/skill-schemas'
+import type { SkillConfigSchema } from '@/schemas/skill-schemas'
 
 import type { IntentObject } from '@sdk/types'
 
@@ -10,15 +9,30 @@ const {
   argv: [, , INTENT_OBJ_FILE_PATH]
 } = process
 
+export const LEON_VERSION = process.env['npm_package_version']
+
+const BRIDGES_PATH = path.join(process.cwd(), 'bridges')
+const NODEJS_BRIDGE_ROOT_PATH = path.join(BRIDGES_PATH, 'nodejs')
+const NODEJS_BRIDGE_SRC_PATH = path.join(NODEJS_BRIDGE_ROOT_PATH, 'src')
+const NODEJS_BRIDGE_VERSION_FILE_PATH = path.join(
+  NODEJS_BRIDGE_SRC_PATH,
+  'version.ts'
+)
+
+export const [, NODEJS_BRIDGE_VERSION] = fs
+  .readFileSync(NODEJS_BRIDGE_VERSION_FILE_PATH, 'utf8')
+  .split("'")
+
 export const INTENT_OBJECT: IntentObject = JSON.parse(
   fs.readFileSync(INTENT_OBJ_FILE_PATH as string, 'utf8')
 )
+
+export const SKILLS_PATH = path.join(process.cwd(), 'skills')
 export const SKILL_PATH = path.join(
-  SKILLS_ROOT_PATH,
+  SKILLS_PATH,
   INTENT_OBJECT.domain,
   INTENT_OBJECT.skill
 )
-export const SKILLS_PATH = SKILLS_ROOT_PATH
 export const SKILL_CONFIG: SkillConfigSchema = JSON.parse(
   fs.readFileSync(
     path.join(
@@ -29,4 +43,3 @@ export const SKILL_CONFIG: SkillConfigSchema = JSON.parse(
     'utf8'
   )
 )
-export { LEON_VERSION, NODEJS_BRIDGE_VERSION } from '@/constants'

bridges/nodejs/src/main.ts

@@ -9,6 +9,7 @@ import { INTENT_OBJECT } from '@bridge/constants'
     action,
     lang,
     utterance,
+    new_utterance,
     current_entities,
     entities,
     current_resolvers,
@@ -20,6 +21,7 @@ import { INTENT_OBJECT } from '@bridge/constants'
   const params: ActionParams = {
     lang,
     utterance,
+    new_utterance,
     current_entities,
     entities,
     current_resolvers,

bridges/nodejs/src/sdk/aurora/button.ts

@@ -1,12 +1,9 @@
-import { Widget } from '../widget'
+import { type ButtonProps } from '@leon-ai/aurora'
 
-// TODO: contains the button API. rendering engine <-> SDK
-interface ButtonOptions {
-  text: string
-}
+import { WidgetComponent } from '../widget-component'
 
-export class Button extends Widget<ButtonOptions> {
-  public constructor(options: ButtonOptions) {
-    super(options)
+export class Button extends WidgetComponent<ButtonProps> {
+  constructor(props: ButtonProps) {
+    super(props)
   }
 }

bridges/nodejs/src/sdk/aurora/card.ts

@@ -0,0 +1,9 @@
+import { type CardProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Card extends WidgetComponent<CardProps> {
+  constructor(props: CardProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/checkbox.ts

@@ -0,0 +1,9 @@
+import { type CheckboxProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Checkbox extends WidgetComponent<CheckboxProps> {
+  constructor(props: CheckboxProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/circular-progress.ts

@@ -0,0 +1,9 @@
+import { type CircularProgressProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class CircularProgress extends WidgetComponent<CircularProgressProps> {
+  constructor(props: CircularProgressProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/flexbox.ts

@@ -0,0 +1,9 @@
+import { type FlexboxProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Flexbox extends WidgetComponent<FlexboxProps> {
+  constructor(props: FlexboxProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/form.ts

@@ -0,0 +1,9 @@
+import { type FormProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Form extends WidgetComponent<FormProps> {
+  constructor(props: FormProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/icon-button.ts

@@ -0,0 +1,9 @@
+import { type IconButtonProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class IconButton extends WidgetComponent<IconButtonProps> {
+  constructor(props: IconButtonProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/icon.ts

@@ -0,0 +1,9 @@
+import { type IconProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Icon extends WidgetComponent<IconProps> {
+  constructor(props: IconProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/image.ts

@@ -0,0 +1,9 @@
+import { type ImageProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Image extends WidgetComponent<ImageProps> {
+  constructor(props: ImageProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/index.ts

@@ -0,0 +1,30 @@
+export * from './button'
+export * from './card'
+export * from './checkbox'
+export * from './circular-progress'
+export * from './flexbox'
+export * from './form'
+export * from './icon'
+export * from './icon-button'
+export * from './image'
+export * from './input'
+export * from './link'
+export * from './list'
+export * from './list-header'
+export * from './list-item'
+export * from './loader'
+export * from './progress'
+export * from './radio'
+export * from './radio-group'
+export * from './range-slider'
+export * from './scroll-container'
+export * from './select'
+export * from './select-option'
+export * from './status'
+export * from './switch'
+export * from './tab'
+export * from './tab-content'
+export * from './tab-group'
+export * from './tab-list'
+export * from './text'
+export * from './widget-wrapper'

bridges/nodejs/src/sdk/aurora/input.ts

@@ -0,0 +1,9 @@
+import { type InputProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Input extends WidgetComponent<InputProps> {
+  constructor(props: InputProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/link.ts

@@ -0,0 +1,9 @@
+import { type LinkProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Link extends WidgetComponent<LinkProps> {
+  constructor(props: LinkProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list-header.ts

@@ -0,0 +1,9 @@
+import { type ListHeaderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ListHeader extends WidgetComponent<ListHeaderProps> {
+  constructor(props: ListHeaderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list-item.ts

@@ -0,0 +1,9 @@
+import { type ListItemProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ListItem extends WidgetComponent<ListItemProps> {
+  constructor(props: ListItemProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/list.ts

@@ -0,0 +1,9 @@
+import { type ListProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class List extends WidgetComponent<ListProps> {
+  constructor(props: ListProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/loader.ts

@@ -0,0 +1,9 @@
+import { type LoaderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Loader extends WidgetComponent<LoaderProps> {
+  constructor(props: LoaderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/progress.ts

@@ -0,0 +1,9 @@
+import { type ProgressProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Progress extends WidgetComponent<ProgressProps> {
+  constructor(props: ProgressProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/radio-group.ts

@@ -0,0 +1,9 @@
+import { type RadioGroupProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class RadioGroup extends WidgetComponent<RadioGroupProps> {
+  constructor(props: RadioGroupProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/radio.ts

@@ -0,0 +1,9 @@
+import { type RadioProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Radio extends WidgetComponent<RadioProps> {
+  constructor(props: RadioProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/range-slider.ts

@@ -0,0 +1,9 @@
+import { type RangeSliderProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class RangeSlider extends WidgetComponent<RangeSliderProps> {
+  constructor(props: RangeSliderProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/scroll-container.ts

@@ -0,0 +1,9 @@
+import { type ScrollContainerProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class ScrollContainer extends WidgetComponent<ScrollContainerProps> {
+  constructor(props: ScrollContainerProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/select-option.ts

@@ -0,0 +1,9 @@
+import { type SelectOptionProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class SelectOption extends WidgetComponent<SelectOptionProps> {
+  constructor(props: SelectOptionProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/select.ts

@@ -0,0 +1,9 @@
+import { type SelectProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Select extends WidgetComponent<SelectProps> {
+  constructor(props: SelectProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/status.ts

@@ -0,0 +1,9 @@
+import { type StatusProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Status extends WidgetComponent<StatusProps> {
+  constructor(props: StatusProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/switch.ts

@@ -0,0 +1,9 @@
+import { type SwitchProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Switch extends WidgetComponent<SwitchProps> {
+  constructor(props: SwitchProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-content.ts

@@ -0,0 +1,9 @@
+import { type TabContentProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabContent extends WidgetComponent<TabContentProps> {
+  constructor(props: TabContentProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-group.ts

@@ -0,0 +1,9 @@
+import { type TabGroupProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabGroup extends WidgetComponent<TabGroupProps> {
+  constructor(props: TabGroupProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab-list.ts

@@ -0,0 +1,9 @@
+import { type TabListProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class TabList extends WidgetComponent<TabListProps> {
+  constructor(props: TabListProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/tab.ts

@@ -0,0 +1,9 @@
+import { type TabProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Tab extends WidgetComponent<TabProps> {
+  constructor(props: TabProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/text.ts

@@ -0,0 +1,9 @@
+import { type TextProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class Text extends WidgetComponent<TextProps> {
+  constructor(props: TextProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/aurora/widget-wrapper.ts

@@ -0,0 +1,9 @@
+import { type WidgetWrapperProps } from '@leon-ai/aurora'
+
+import { WidgetComponent } from '../widget-component'
+
+export class WidgetWrapper extends WidgetComponent<WidgetWrapperProps> {
+  constructor(props: WidgetWrapperProps) {
+    super(props)
+  }
+}

bridges/nodejs/src/sdk/leon.ts

@@ -5,6 +5,8 @@ import type {
   AnswerConfig
 } from '@sdk/types'
 import { INTENT_OBJECT, SKILL_CONFIG } from '@bridge/constants'
+import { WidgetWrapper } from '@sdk/aurora'
+import { SUPPORTED_WIDGET_EVENTS } from '@sdk/widget-component'
 
 class Leon {
   private static instance: Leon
@@ -42,18 +44,26 @@ class Leon {
 
       if (data != null) {
         for (const key in data) {
-          // In case the answer needs speech and text differentiation
-          if (typeof answer !== 'string' && answer.text) {
-            answer.text = answer.text.replaceAll(`%${key}%`, String(data[key]))
-            answer.speech = answer.speech.replaceAll(
-              `%${key}%`,
-              String(data[key])
-            )
-          } else {
+          if (typeof answer === 'string') {
             answer = (answer as string).replaceAll(
               `%${key}%`,
               String(data[key])
             )
+          } else {
+            // In case the answer needs speech and text differentiation
+
+            if (answer.text) {
+              answer.text = answer.text.replaceAll(
+                `%${key}%`,
+                String(data[key])
+              )
+            }
+            if (answer.speech) {
+              answer.speech = answer.speech.replaceAll(
+                `%${key}%`,
+                String(data[key])
+              )
+            }
           }
         }
       }
@@ -62,21 +72,26 @@ class Leon {
         const { variables } = SKILL_CONFIG
 
         for (const key in variables) {
-          // In case the answer needs speech and text differentiation
-          if (typeof answer !== 'string' && answer.text) {
-            answer.text = answer.text.replaceAll(
-              `%${key}%`,
-              String(variables[key])
-            )
-            answer.speech = answer.speech.replaceAll(
-              `%${key}%`,
-              String(variables[key])
-            )
-          } else {
+          if (typeof answer === 'string') {
             answer = (answer as string).replaceAll(
               `%${key}%`,
               String(variables[key])
             )
+          } else {
+            // In case the answer needs speech and text differentiation
+
+            if (answer.text) {
+              answer.text = answer.text.replaceAll(
+                `%${key}%`,
+                String(variables[key])
+              )
+            }
+            if (answer.speech) {
+              answer.speech = answer.speech.replaceAll(
+                `%${key}%`,
+                String(variables[key])
+              )
+            }
           }
         }
       }
@@ -114,7 +129,17 @@ class Leon {
       }
 
       if (answerInput.widget) {
-        answerObject.output.widget = answerInput.widget
+        answerObject.output.widget = {
+          actionName: `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${INTENT_OBJECT.action}`,
+          widget: answerInput.widget.widget,
+          id: answerInput.widget.id,
+          onFetch: answerInput.widget.onFetch ?? null,
+          componentTree: new WidgetWrapper({
+            ...answerInput.widget.wrapperProps,
+            children: [answerInput.widget.render()]
+          }),
+          supportedEvents: SUPPORTED_WIDGET_EVENTS
+        }
       }
 
       // "Temporize" for the data buffer output on the core

bridges/nodejs/src/sdk/memory.ts

@@ -24,6 +24,7 @@ export class Memory<T = unknown> {
 
     if (this.name.includes(':') && this.name.split(':').length === 3) {
       this.isFromAnotherSkill = true
+
       const [domainName, skillName, memoryName] = this.name.split(':')
       this.memoryPath = path.join(
         SKILLS_PATH,

bridges/nodejs/src/sdk/network.ts

@@ -38,7 +38,7 @@ interface NetworkResponse<ResponseData> {
 export class NetworkError<ResponseErrorData = unknown> extends Error {
   public readonly response: NetworkResponse<ResponseErrorData>
 
-  public constructor(response: NetworkResponse<ResponseErrorData>) {
+  constructor(response: NetworkResponse<ResponseErrorData>) {
     super(`[NetworkError]: ${response.statusCode} ${response.data}`)
     this.response = response
     Object.setPrototypeOf(this, NetworkError.prototype)
@@ -49,7 +49,7 @@ export class Network {
   private readonly options: NetworkOptions
   private axios: AxiosInstance
 
-  public constructor(options: NetworkOptions = {}) {
+  constructor(options: NetworkOptions = {}) {
     this.options = options
     this.axios = axios.create({
       baseURL: this.options.baseURL

bridges/nodejs/src/sdk/settings.ts

@@ -7,7 +7,7 @@ export class Settings<T extends Record<string, unknown>> {
   private readonly settingsPath: string
   private readonly settingsSamplePath: string
 
-  public constructor() {
+  constructor() {
     this.settingsPath = path.join(SKILL_PATH, 'src', 'settings.json')
     this.settingsSamplePath = path.join(
       SKILL_PATH,

bridges/nodejs/src/sdk/toolbox.ts

@@ -0,0 +1,13 @@
+import { INTENT_OBJECT } from '@bridge/constants'
+
+/**
+ * Get the widget id if any
+ * @example getWidgetId() // 'timerwidget-5q1xlzeh
+ */
+export function getWidgetId(): string | null {
+  return (
+    INTENT_OBJECT.current_entities.find(
+      (entity) => entity.entity === 'widgetid'
+    )?.sourceText ?? null
+  )
+}

bridges/nodejs/src/sdk/types.ts

@@ -9,6 +9,8 @@ import type {
 } from '@/core/brain/types'
 import type { SkillAnswerConfigSchema } from '@/schemas/skill-schemas'
 
+import type { Widget } from '@sdk/widget'
+
 export type { ActionParams, IntentObject }
 
 export * from '@/core/nlp/types'
@@ -20,7 +22,7 @@ export type ActionFunction = (params: ActionParams) => Promise<void>
  */
 export interface Answer {
   key?: string
-  widget?: unknown // TODO
+  widget?: Widget
   data?: AnswerData
   core?: SkillAnswerCoreData
 }
@@ -28,7 +30,7 @@ export interface TextAnswer extends Answer {
   key: string
 }
 export interface WidgetAnswer extends Answer {
-  widget: unknown
+  widget: Widget
   key?: string
 }
 export type AnswerData = Record<string, string | number> | null

bridges/nodejs/src/sdk/widget-component.ts

@@ -0,0 +1,51 @@
+export type SupportedWidgetEvent = (typeof SUPPORTED_WIDGET_EVENTS)[number]
+interface WidgetEvent {
+  type: SupportedWidgetEvent
+  id: string
+}
+
+export const SUPPORTED_WIDGET_EVENTS = [
+  'onClick',
+  'onSubmit',
+  'onChange',
+  'onStart',
+  'onEnd'
+] as const
+
+function generateId(): string {
+  return Math.random().toString(36).substring(2, 7)
+}
+
+export abstract class WidgetComponent<T = unknown> {
+  public readonly component: string
+  public readonly id: string
+  public readonly props: T
+  public readonly events: WidgetEvent[]
+
+  protected constructor(props: T) {
+    this.component = this.constructor.name
+    this.id = `${this.component.toLowerCase()}-${generateId()}`
+    this.props = props
+    this.events = this.parseEvents()
+  }
+
+  private parseEvents(): WidgetEvent[] {
+    if (!this.props) {
+      return []
+    }
+
+    const eventTypes = Object.keys(this.props).filter(
+      (key) =>
+        key.startsWith('on') &&
+        SUPPORTED_WIDGET_EVENTS.includes(key as SupportedWidgetEvent)
+    ) as SupportedWidgetEvent[]
+
+    return eventTypes.map((type) => ({
+      type,
+      id: `${this.id}_${type.toLowerCase()}-${generateId()}`,
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-expect-error
+      method: this.props[type]()
+    }))
+  }
+}

bridges/nodejs/src/sdk/widget.ts

@@ -1,9 +1,138 @@
-export abstract class Widget<T> {
-  public readonly type: string
-  public readonly options: T
+import { type WidgetWrapperProps } from '@leon-ai/aurora'
 
-  protected constructor(options: T) {
-    this.type = this.constructor.name
-    this.options = options
+import { INTENT_OBJECT, SKILL_CONFIG } from '@bridge/constants'
+import { WidgetComponent } from '@sdk/widget-component'
+
+type UtteranceSender = 'leon' | 'owner'
+
+interface SendUtteranceWidgetEventMethodParams {
+  from: UtteranceSender
+  utterance: string
+}
+interface RunSkillActionWidgetEventMethodParams {
+  actionName: string
+  params: Record<string, unknown>
+}
+interface SendUtteranceOptions {
+  from?: UtteranceSender
+  data?: Record<string, unknown>
+}
+
+export interface WidgetEventMethod {
+  methodName: 'send_utterance' | 'run_skill_action'
+  methodParams:
+    | SendUtteranceWidgetEventMethodParams
+    | RunSkillActionWidgetEventMethodParams
+}
+export interface WidgetOptions<T = unknown> {
+  wrapperProps?: Omit<WidgetWrapperProps, 'children'>
+  onFetch?: {
+    widgetId?: string | undefined
+    actionName: string
+  }
+  params: T
+}
+
+export abstract class Widget<T = unknown> {
+  public actionName: string
+  public id: string
+  public widget: string
+  public onFetch: WidgetOptions<T>['onFetch'] | null = null
+  public wrapperProps: WidgetOptions<T>['wrapperProps']
+  public params: WidgetOptions<T>['params']
+
+  protected constructor(options: WidgetOptions<T>) {
+    if (options?.wrapperProps) {
+      this.wrapperProps = options.wrapperProps
+    }
+    this.actionName = `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${INTENT_OBJECT.action}`
+    this.params = options.params
+    this.widget = this.constructor.name
+    if (options?.onFetch) {
+      this.onFetch = {
+        widgetId: options.onFetch.widgetId,
+        actionName: `${INTENT_OBJECT.domain}:${INTENT_OBJECT.skill}:${options.onFetch.actionName}`
+      }
+    }
+    this.id =
+      options.onFetch?.widgetId ||
+      `${this.widget.toLowerCase()}-${Math.random()
+        .toString(36)
+        .substring(2, 10)}`
+  }
+
+  /**
+   * Render the widget
+   */
+  public abstract render(): WidgetComponent<unknown>
+
+  /**
+   * Indicate the core to send a given utterance
+   * @param key The key of the content
+   * @param options The options of the utterance
+   * @example content('provider_selected', { data: { provider: 'Spotify' } }) // 'I chose the Spotify provider'
+   */
+  protected sendUtterance(
+    key: string,
+    options?: SendUtteranceOptions
+  ): WidgetEventMethod {
+    const utteranceContent = this.content(key, options?.data)
+    const from = options?.from || 'owner'
+
+    return {
+      methodName: 'send_utterance',
+      methodParams: {
+        from,
+        utterance: utteranceContent
+      }
+    }
+  }
+
+  /**
+   * Indicate the core to run a given skill action
+   * @param actionName The name of the action
+   * @param params The parameters of the action
+   * @example runSkillAction('music_audio:player:next', { provider: 'Spotify' })
+   */
+  protected runSkillAction(
+    actionName: string,
+    params: Record<string, unknown>
+  ): WidgetEventMethod {
+    return {
+      methodName: 'run_skill_action',
+      methodParams: {
+        actionName,
+        params
+      }
+    }
+  }
+
+  /**
+   * Grab and compute the target content of the widget
+   * @param key The key of the content
+   * @param data The data to apply
+   * @example content('select_provider') // 'Please select a provider'
+   * @example content('provider_selected', { provider: 'Spotify' }) // 'I chose the Spotify provider'
+   */
+  protected content(key: string, data?: Record<string, unknown>): string {
+    const { widget_contents: widgetContents } = SKILL_CONFIG
+
+    if (!widgetContents || !widgetContents[key]) {
+      return 'INVALID'
+    }
+
+    let content = widgetContents[key]
+
+    if (Array.isArray(content)) {
+      content = content[Math.floor(Math.random() * content.length)] as string
+    }
+
+    if (data) {
+      for (const key in data) {
+        content = content.replaceAll(`%${key}%`, String(data[key]))
+      }
+    }
+
+    return content
   }
 }

bridges/nodejs/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.1.0'
+export const VERSION = '1.2.0'

bridges/nodejs/tsconfig.json

@@ -14,11 +14,6 @@
     "exactOptionalPropertyTypes": false,
     "declaration": true
   },
-  "ts-node": {
-    "swc": true,
-    "require": ["tsconfig-paths/register"],
-    "files": true
-  },
   "include": ["src/**/*"],
   "exclude": ["node_modules", "dist"]
 }

bridges/python/src/Pipfile

@@ -4,11 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [requires]
-python_version = "3.9.10"
+python_version = "3.11.9"
 
 [packages]
 setuptools = "*"
 wheel = "*"
-cx-freeze = "==6.11.1"
-requests = "==2.28.1"
+cx-freeze = "==7.1.1"
+requests = "==2.32.3"
 beautifulsoup4 = "==4.7.1"

bridges/python/src/main.py

@@ -9,6 +9,7 @@ def main():
     params = {
         'lang': INTENT_OBJECT['lang'],
         'utterance': INTENT_OBJECT['utterance'],
+        'new_utterance': INTENT_OBJECT['new_utterance'],
         'current_entities': INTENT_OBJECT['current_entities'],
         'entities': INTENT_OBJECT['entities'],
         'current_resolvers': INTENT_OBJECT['current_resolvers'],

bridges/python/src/sdk/aurora/button.py

@@ -1,9 +1,6 @@
-from ..widget import Widget
+from ..widget_component import WidgetComponent
 
 
-# TODO: contains the button API. rendering engine <-> SDK
-
-
-class Button(Widget):
-    def __init__(self, options):
-        super().__init__(options)
+class Button(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/card.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Card(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/checkbox.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Checkbox(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/circular_progress.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class CircularProgress(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/flexbox.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Flexbox(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/form.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Form(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/icon.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Icon(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/icon_button.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class IconButton(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/image.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Image(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/input.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Input(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/link.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Link(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/list.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class List(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/list_header.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class ListHeader(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/list_item.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class ListItem(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/loader.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Loader(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/progress.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Progress(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/radio.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class Radio(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/radio_group.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class RadioGroup(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)

bridges/python/src/sdk/aurora/range_slider.py

@@ -0,0 +1,6 @@
+from ..widget_component import WidgetComponent
+
+
+class RangeSlider(WidgetComponent[dict]):
+    def __init__(self, props: dict):
+        super().__init__(props)