Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9c5d2ea0c8 | |||
| b7a4f69565 | |||
| 7b8a324658 | |||
| 59cfded2bb | |||
| 8dc8c05d7d | |||
| 0e4ec4fb97 | |||
| bb00677568 | |||
| 308b63f576 | |||
| 40819e8e17 | |||
| 0fba50905f | |||
| 419bfe3d78 | |||
| 1847248f12 | |||
| eed4a9597c | |||
| c3452d2170 | |||
| 0a14567776 | |||
| 71573b78d1 | |||
| 722dba875b | |||
| 593419639e | |||
| 8b2f06e305 | |||
| cb2db411ce | |||
| f179e1070c | |||
| fdeff12762 |
@@ -123,11 +123,14 @@ Types define which kind of changes you made to the project.
|
||||
|
||||
Scopes define high-level nodes of Leon.
|
||||
|
||||
- web app
|
||||
- bridge/python
|
||||
- bridge/nodejs
|
||||
- docker
|
||||
- hotword
|
||||
- scripts
|
||||
- server
|
||||
- tcp server
|
||||
- python bridge
|
||||
- hotword
|
||||
- web app
|
||||
- skill/skill_name
|
||||
|
||||
### Examples
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
name: Pre-release Node.js bridge
|
||||
|
||||
on: workflow_dispatch
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Install Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: lts/*
|
||||
|
||||
- name: Set Node.js bridge version
|
||||
working-directory: bridges/nodejs/src
|
||||
run: |
|
||||
echo "NODEJS_BRIDGE_VERSION=$(node --require fs --eval "const fs = require('node:fs'); const [, VERSION] = fs.readFileSync('version.ts', 'utf8').split(\"'\"); console.log(VERSION)")" >> $GITHUB_ENV
|
||||
|
||||
- name: Display Node.js bridge version
|
||||
run: |
|
||||
echo "Node.js bridge version: ${{ env.NODEJS_BRIDGE_VERSION }}"
|
||||
|
||||
- name: Install core
|
||||
run: npm install
|
||||
|
||||
- name: Build Node.js bridge
|
||||
run: npm run build:nodejs-bridge
|
||||
|
||||
- name: Upload Node.js bridge
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
path: bridges/nodejs/dist/*.zip
|
||||
|
||||
draft-release:
|
||||
name: Draft-release
|
||||
needs: [build]
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Install Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: lts/*
|
||||
|
||||
- name: Set Node.js bridge version
|
||||
working-directory: bridges/nodejs/src
|
||||
run: |
|
||||
echo "NODEJS_BRIDGE_VERSION=$(node --require fs --eval "const fs = require('node:fs'); const [, VERSION] = fs.readFileSync('version.ts', 'utf8').split(\"'\"); console.log(VERSION)")" >> $GITHUB_ENV
|
||||
|
||||
- name: Download Node.js bridge
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
path: bridges/nodejs/dist
|
||||
|
||||
- uses: marvinpinto/action-automatic-releases@latest
|
||||
with:
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
automatic_release_tag: nodejs-bridge_v${{ env.NODEJS_BRIDGE_VERSION }}
|
||||
draft: true
|
||||
prerelease: false
|
||||
title: Node.js Bridge ${{ env.NODEJS_BRIDGE_VERSION }}
|
||||
files: bridges/nodejs/dist/artifact/*.zip
|
||||
@@ -26,6 +26,7 @@ bridges/python/src/Pipfile.lock
|
||||
tcp_server/src/Pipfile.lock
|
||||
!tcp_server/**/.gitkeep
|
||||
!bridges/python/**/.gitkeep
|
||||
!bridges/nodejs/**/.gitkeep
|
||||
!**/*.sample*
|
||||
packages/**/config/config.json
|
||||
skills/**/src/config.json
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# [1.0.0-beta.8](https://github.com/leon-ai/leon/compare/v1.0.0-beta.7...v1.0.0-beta.8) (2023-05-01) / Binaries and TypeScript Rewrite
|
||||
|
||||
_Please refer to [our latest blog post](https://blog.getleon.ai/binaries-and-typescript-rewrite-1-0-0-beta-8/) for more information on the new release of our dear Leon._
|
||||
|
||||
# [1.0.0-beta.7](https://github.com/leon-ai/leon/compare/v1.0.0-beta.6...v1.0.0-beta.7) (2022-08-24) / A Much Better NLP
|
||||
|
||||
_Please [read this blog post](https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/) to know more about all the new features and the exciting future of Leon._
|
||||
|
||||
@@ -56,18 +56,19 @@ Here is how LLMs may help Leon in the future:
|
||||
|
||||
### What's Next?
|
||||
|
||||
Once the new core released, we'll work on the community aspect of Leon. For example, better organize our [Discord](https://discord.gg/MNQqqKg), planify regular calls, work on skills together, etc. It is very important for Leon to have a real community. At that moment, the skills platform will already be online, so it'll be easier to sync our progress and publish new skills.
|
||||
Once the new core released, we'll work on the community aspect of Leon. For example, better organize [our Discord](https://discord.gg/MNQqqKg), planify regular calls, work on skills together, etc. It is very important for Leon to have a real community. At that moment, the skills platform will already be online, so it'll be easier to sync our progress and publish new skills.
|
||||
|
||||
- Feel free to check out the Git development branches and our [next major milestones](https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/#whats-next).
|
||||
- And the [detailed roadmap](http://roadmap.getleon.ai).
|
||||
- Many exciting things are coming up, hence no new documentation and test are going to be written until the official release of Leon.
|
||||
|
||||
<h2 align="center">📢 Notice 📢</h2>
|
||||
<p align="center">
|
||||
<a href="https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/"><img width="400" src="https://blog.getleon.ai/static/a62ac28a01cb6898e299dced40875a68/c1b63/beta-7.png" /></a>
|
||||
<br>
|
||||
Many exciting things are coming up, hence no new documentation and test are going to be written until the official release of Leon. Feel free to <a href="https://discord.gg/MNQqqKg"><b>join us on Discord</b></a> to know more and to read the <a href="https://blog.getleon.ai/a-much-better-nlp-and-future-1-0-0-beta-7/"><b>"A Much Better NLP and Future" blog post</b></a>.
|
||||
</p>
|
||||
<br><br>
|
||||
---
|
||||
|
||||
## Latest Release
|
||||
|
||||
Check out the [latest release blog post](https://blog.getleon.ai/binaries-and-typescript-rewrite-1-0-0-beta-8/).
|
||||
|
||||
<a href="https://blog.getleon.ai/binaries-and-typescript-rewrite-1-0-0-beta-8/"><img width="400" src="https://blog.getleon.ai/static/a0d1cbafd1968e7531dc17e229f8cc61/aa440/beta-8.png" /></a>
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
package-lock=false
|
||||
save-exact=true
|
||||
Vendored
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "leon-nodejs-bridge",
|
||||
"description": "Leon's Node.js bridge to communicate between the core and skills made with JavaScript",
|
||||
"main": "dist/leon-nodejs-bridge.js",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
"email": "louis@getleon.ai",
|
||||
"url": "https://twitter.com/grenlouis"
|
||||
},
|
||||
"license": "MIT",
|
||||
"homepage": "https://getleon.ai",
|
||||
"bugs": {
|
||||
"url": "https://github.com/leon-ai/leon/issues"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { VERSION } from './version'
|
||||
|
||||
console.log('[WIP] Node.js bridge', VERSION)
|
||||
|
||||
// TODO
|
||||
@@ -0,0 +1 @@
|
||||
export const VERSION = '0.0.0'
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"extends": "@tsconfig/node16-strictest/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"ignoreDeprecations": "5.0",
|
||||
"allowJs": true,
|
||||
"checkJs": false,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"ts-node": {
|
||||
"swc": true,
|
||||
"require": ["tsconfig-paths/register"],
|
||||
"files": true
|
||||
},
|
||||
"files": [],
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["dist"]
|
||||
}
|
||||
+4
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "leon",
|
||||
"version": "1.0.0-beta.8+dev",
|
||||
"version": "1.0.0-beta.8",
|
||||
"description": "Server, skills and web app of the Leon personal assistant",
|
||||
"author": {
|
||||
"name": "Louis Grenard",
|
||||
@@ -50,7 +50,8 @@
|
||||
"generate:json-schemas": "ts-node scripts/generate/run-generate-json-schemas.js",
|
||||
"build": "npm run build:app && npm run build:server",
|
||||
"build:app": "cross-env LEON_NODE_ENV=production ts-node scripts/app/run-build-app.js",
|
||||
"build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && tsc && resolve-tspaths && shx rm -rf server/dist/core server/dist/package.json && shx mv -f server/dist/server/src/* server/dist && shx rm -rf server/dist/server && shx mkdir -p server/dist/tmp",
|
||||
"build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && tsc --project tsconfig.json && resolve-tspaths && shx rm -rf server/dist/core server/dist/package.json && shx mv -f server/dist/server/src/* server/dist && shx rm -rf server/dist/server && shx mkdir -p server/dist/tmp",
|
||||
"build:nodejs-bridge": "ts-node scripts/build-binaries.js nodejs-bridge",
|
||||
"build:python-bridge": "ts-node scripts/build-binaries.js python-bridge",
|
||||
"build:tcp-server": "ts-node scripts/build-binaries.js tcp-server",
|
||||
"start:tcp-server": "cross-env PIPENV_PIPFILE=tcp_server/src/Pipfile pipenv run python tcp_server/src/main.py",
|
||||
@@ -58,6 +59,7 @@
|
||||
"python-bridge": "cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run python bridges/python/src/main.py server/src/intent-object.sample.json",
|
||||
"train": "ts-node scripts/train/run-train.js",
|
||||
"prepare-release": "ts-node scripts/release/prepare-release.js",
|
||||
"pre-release:nodejs-bridge": "ts-node scripts/release/pre-release-binaries.js nodejs-bridge",
|
||||
"pre-release:python-bridge": "ts-node scripts/release/pre-release-binaries.js python-bridge",
|
||||
"pre-release:tcp-server": "ts-node scripts/release/pre-release-binaries.js tcp-server",
|
||||
"check": "ts-node scripts/check.js",
|
||||
|
||||
+69
-23
@@ -9,10 +9,13 @@ import {
|
||||
PYTHON_BRIDGE_SRC_PATH,
|
||||
TCP_SERVER_SRC_PATH,
|
||||
BINARIES_FOLDER_NAME,
|
||||
NODEJS_BRIDGE_DIST_PATH,
|
||||
PYTHON_BRIDGE_DIST_PATH,
|
||||
TCP_SERVER_DIST_PATH,
|
||||
NODEJS_BRIDGE_BIN_NAME,
|
||||
PYTHON_BRIDGE_BIN_NAME,
|
||||
TCP_SERVER_BIN_NAME
|
||||
TCP_SERVER_BIN_NAME,
|
||||
NODEJS_BRIDGE_ROOT_PATH
|
||||
} from '@/constants'
|
||||
import { OSTypes } from '@/types'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
@@ -29,8 +32,15 @@ import { SystemHelper } from '@/helpers/system-helper'
|
||||
|
||||
const BUILD_TARGETS = new Map()
|
||||
|
||||
BUILD_TARGETS.set('nodejs-bridge', {
|
||||
name: 'Node.js bridge',
|
||||
needsPythonEnv: false,
|
||||
distPath: NODEJS_BRIDGE_DIST_PATH,
|
||||
archiveName: `${NODEJS_BRIDGE_BIN_NAME.split('.')[0]}.zip`
|
||||
})
|
||||
BUILD_TARGETS.set('python-bridge', {
|
||||
name: 'Python bridge',
|
||||
needsPythonEnv: true,
|
||||
pipfilePath: path.join(PYTHON_BRIDGE_SRC_PATH, 'Pipfile'),
|
||||
setupFilePath: path.join(PYTHON_BRIDGE_SRC_PATH, 'setup.py'),
|
||||
distPath: PYTHON_BRIDGE_DIST_PATH,
|
||||
@@ -39,6 +49,7 @@ BUILD_TARGETS.set('python-bridge', {
|
||||
})
|
||||
BUILD_TARGETS.set('tcp-server', {
|
||||
name: 'TCP server',
|
||||
needsPythonEnv: true,
|
||||
pipfilePath: path.join(TCP_SERVER_SRC_PATH, 'Pipfile'),
|
||||
setupFilePath: path.join(TCP_SERVER_SRC_PATH, 'setup.py'),
|
||||
distPath: TCP_SERVER_DIST_PATH,
|
||||
@@ -62,13 +73,16 @@ BUILD_TARGETS.set('tcp-server', {
|
||||
|
||||
const {
|
||||
name: buildTarget,
|
||||
needsPythonEnv,
|
||||
pipfilePath,
|
||||
setupFilePath,
|
||||
distPath,
|
||||
archiveName,
|
||||
dotVenvPath
|
||||
} = BUILD_TARGETS.get(givenBuildTarget)
|
||||
const buildPath = path.join(distPath, BINARIES_FOLDER_NAME)
|
||||
const buildPath = needsPythonEnv
|
||||
? path.join(distPath, BINARIES_FOLDER_NAME)
|
||||
: distPath
|
||||
|
||||
const { type: osType } = SystemHelper.getInformation()
|
||||
|
||||
@@ -76,7 +90,7 @@ BUILD_TARGETS.set('tcp-server', {
|
||||
* Install requirements
|
||||
*/
|
||||
try {
|
||||
if (osType === OSTypes.Linux) {
|
||||
if (needsPythonEnv && osType === OSTypes.Linux) {
|
||||
LogHelper.info('Checking whether the "patchelf" utility can be found...')
|
||||
|
||||
await command('patchelf --version', { shell: true })
|
||||
@@ -92,30 +106,58 @@ BUILD_TARGETS.set('tcp-server', {
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Build
|
||||
*/
|
||||
try {
|
||||
LogHelper.info(`Building the ${buildTarget}...`)
|
||||
LogHelper.info(`Building the ${buildTarget}...`)
|
||||
|
||||
// Required environment variables to set up
|
||||
process.env.PIPENV_PIPFILE = pipfilePath
|
||||
process.env.PIPENV_VENV_IN_PROJECT = true
|
||||
if (needsPythonEnv) {
|
||||
/**
|
||||
* Build for binaries requiring a Python environment
|
||||
*/
|
||||
try {
|
||||
// Required environment variables to set up
|
||||
process.env.PIPENV_PIPFILE = pipfilePath
|
||||
process.env.PIPENV_VENV_IN_PROJECT = true
|
||||
|
||||
await command(
|
||||
`pipenv run python ${setupFilePath} build --build-exe ${buildPath}`,
|
||||
{
|
||||
await command(
|
||||
`pipenv run python ${setupFilePath} build --build-exe ${buildPath}`,
|
||||
{
|
||||
shell: true,
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
|
||||
LogHelper.success(`The ${buildTarget} has been built`)
|
||||
} catch (e) {
|
||||
LogHelper.error(
|
||||
`An error occurred while building the ${buildTarget}. Try to delete the ${dotVenvPath} folder, run the setup command then build again: ${e}`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
} else {
|
||||
/**
|
||||
* Build for binaries not requiring a Python environment
|
||||
*/
|
||||
try {
|
||||
const tsconfigPath = path.join(NODEJS_BRIDGE_ROOT_PATH, 'tsconfig.json')
|
||||
const distMainFilePath = path.join(NODEJS_BRIDGE_DIST_PATH, 'main.js')
|
||||
const distRenamedMainFilePath = path.join(
|
||||
NODEJS_BRIDGE_DIST_PATH,
|
||||
NODEJS_BRIDGE_BIN_NAME
|
||||
)
|
||||
|
||||
await command(`tsc --project ${tsconfigPath}`, {
|
||||
shell: true,
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
LogHelper.success(`The ${buildTarget} has been built`)
|
||||
} catch (e) {
|
||||
LogHelper.error(
|
||||
`An error occurred while building the ${buildTarget}. Try to delete the ${dotVenvPath} folder, run the setup command then build again: ${e}`
|
||||
)
|
||||
process.exit(1)
|
||||
await fs.promises.rename(distMainFilePath, distRenamedMainFilePath)
|
||||
|
||||
LogHelper.success(`The ${buildTarget} has been built`)
|
||||
} catch (e) {
|
||||
LogHelper.error(
|
||||
`An error occurred while building the ${buildTarget}: ${e}`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -143,7 +185,11 @@ BUILD_TARGETS.set('tcp-server', {
|
||||
|
||||
archive.pipe(output)
|
||||
|
||||
archive.directory(buildPath, BINARIES_FOLDER_NAME)
|
||||
if (needsPythonEnv) {
|
||||
archive.directory(buildPath, BINARIES_FOLDER_NAME)
|
||||
} else {
|
||||
archive.glob(`**/!(${archiveName})`, { cwd: distPath })
|
||||
}
|
||||
|
||||
await archive.finalize()
|
||||
})()
|
||||
|
||||
+36
-34
@@ -14,11 +14,13 @@ import getos from 'getos'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { SystemHelper } from '@/helpers/system-helper'
|
||||
import {
|
||||
MINIMUM_REQUIRED_RAM,
|
||||
LEON_VERSION,
|
||||
PYTHON_BRIDGE_BIN_PATH,
|
||||
TCP_SERVER_BIN_PATH,
|
||||
TCP_SERVER_VERSION,
|
||||
PYTHON_BRIDGE_VERSION
|
||||
PYTHON_BRIDGE_VERSION,
|
||||
INSTANCE_ID
|
||||
} from '@/constants'
|
||||
|
||||
dotenv.config()
|
||||
@@ -31,7 +33,6 @@ dotenv.config()
|
||||
try {
|
||||
const nodeMinRequiredVersion = '16'
|
||||
const npmMinRequiredVersion = '8'
|
||||
const minimumRequiredRAM = 4
|
||||
const flitePath = 'bin/flite/flite'
|
||||
const coquiLanguageModelPath = 'bin/coqui/huge-vocabulary.scorer'
|
||||
const amazonPath = 'core/config/voice/amazon.json'
|
||||
@@ -88,8 +89,9 @@ dotenv.config()
|
||||
v: true
|
||||
}
|
||||
}
|
||||
let pastebinData = {
|
||||
let reportDataInput = {
|
||||
leonVersion: null,
|
||||
instanceID: INSTANCE_ID || null,
|
||||
environment: {
|
||||
osDetails: null,
|
||||
nodeVersion: null,
|
||||
@@ -125,7 +127,7 @@ dotenv.config()
|
||||
|
||||
LogHelper.info('Leon version')
|
||||
LogHelper.success(`${LEON_VERSION}\n`)
|
||||
pastebinData.leonVersion = LEON_VERSION
|
||||
reportDataInput.leonVersion = LEON_VERSION
|
||||
|
||||
/**
|
||||
* Environment checking
|
||||
@@ -144,10 +146,10 @@ dotenv.config()
|
||||
}
|
||||
const totalRAMInGB = SystemHelper.getTotalRAM()
|
||||
|
||||
if (totalRAMInGB < minimumRequiredRAM) {
|
||||
if (Math.round(totalRAMInGB) < MINIMUM_REQUIRED_RAM) {
|
||||
report.can_run.v = false
|
||||
LogHelper.error(
|
||||
`Total RAM: ${totalRAMInGB} GB. Leon needs at least ${minimumRequiredRAM} GB of RAM`
|
||||
`Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM`
|
||||
)
|
||||
} else {
|
||||
LogHelper.success(`Total RAM: ${totalRAMInGB} GB`)
|
||||
@@ -162,8 +164,8 @@ dotenv.config()
|
||||
LogHelper.success(`${JSON.stringify(osInfo)}\n`)
|
||||
}
|
||||
|
||||
pastebinData.environment.osDetails = osInfo
|
||||
pastebinData.environment.totalRAMInGB = totalRAMInGB
|
||||
reportDataInput.environment.osDetails = osInfo
|
||||
reportDataInput.environment.totalRAMInGB = totalRAMInGB
|
||||
;(
|
||||
await Promise.all([
|
||||
command('node --version', { shell: true }),
|
||||
@@ -195,9 +197,9 @@ dotenv.config()
|
||||
} else {
|
||||
LogHelper.success(`${p.stdout}\n`)
|
||||
if (p.command.includes('node --version')) {
|
||||
pastebinData.environment.nodeVersion = p.stdout
|
||||
reportDataInput.environment.nodeVersion = p.stdout
|
||||
} else if (p.command.includes('npm --version')) {
|
||||
pastebinData.environment.npmVersion = p.stdout
|
||||
reportDataInput.environment.npmVersion = p.stdout
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -207,7 +209,7 @@ dotenv.config()
|
||||
*/
|
||||
|
||||
LogHelper.success(`Python bridge version: ${PYTHON_BRIDGE_VERSION}`)
|
||||
pastebinData.pythonBridge.version = PYTHON_BRIDGE_VERSION
|
||||
reportDataInput.pythonBridge.version = PYTHON_BRIDGE_VERSION
|
||||
LogHelper.info('Executing a skill...')
|
||||
|
||||
try {
|
||||
@@ -224,16 +226,16 @@ dotenv.config()
|
||||
const executionEnd = Date.now()
|
||||
const executionTime = executionEnd - executionStart
|
||||
LogHelper.info(p.command)
|
||||
pastebinData.pythonBridge.command = p.command
|
||||
reportDataInput.pythonBridge.command = p.command
|
||||
LogHelper.success(p.stdout)
|
||||
pastebinData.pythonBridge.output = p.stdout
|
||||
reportDataInput.pythonBridge.output = p.stdout
|
||||
LogHelper.info(`Skill execution time: ${executionTime}ms\n`)
|
||||
pastebinData.pythonBridge.executionTime = `${executionTime}ms`
|
||||
reportDataInput.pythonBridge.executionTime = `${executionTime}ms`
|
||||
} catch (e) {
|
||||
LogHelper.info(e.command)
|
||||
report.can_run_skill.v = false
|
||||
LogHelper.error(`${e}\n`)
|
||||
pastebinData.pythonBridge.error = JSON.stringify(e)
|
||||
reportDataInput.pythonBridge.error = JSON.stringify(e)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -241,7 +243,7 @@ dotenv.config()
|
||||
*/
|
||||
|
||||
LogHelper.success(`TCP server version: ${TCP_SERVER_VERSION}`)
|
||||
pastebinData.tcpServer.version = TCP_SERVER_VERSION
|
||||
reportDataInput.tcpServer.version = TCP_SERVER_VERSION
|
||||
|
||||
LogHelper.info('Starting the TCP server...')
|
||||
|
||||
@@ -253,7 +255,7 @@ dotenv.config()
|
||||
]
|
||||
|
||||
LogHelper.info(tcpServerCommand)
|
||||
pastebinData.tcpServer.command = tcpServerCommand
|
||||
reportDataInput.tcpServer.command = tcpServerCommand
|
||||
|
||||
if (osInfo.platform === 'darwin') {
|
||||
LogHelper.info(
|
||||
@@ -280,7 +282,7 @@ dotenv.config()
|
||||
if (!ignoredWarnings.some((w) => newData.includes(w))) {
|
||||
tcpServerOutput += newData
|
||||
report.can_start_tcp_server.v = false
|
||||
pastebinData.tcpServer.error = newData
|
||||
reportDataInput.tcpServer.error = newData
|
||||
LogHelper.error(`Cannot start the TCP server: ${newData}`)
|
||||
}
|
||||
})
|
||||
@@ -292,16 +294,16 @@ dotenv.config()
|
||||
|
||||
const error = `The TCP server timed out after ${timeout}ms`
|
||||
LogHelper.error(error)
|
||||
pastebinData.tcpServer.error = error
|
||||
reportDataInput.tcpServer.error = error
|
||||
report.can_start_tcp_server.v = false
|
||||
}, timeout)
|
||||
|
||||
p.stdout.on('end', async () => {
|
||||
const tcpServerEnd = Date.now()
|
||||
pastebinData.tcpServer.output = tcpServerOutput
|
||||
pastebinData.tcpServer.startTime = `${tcpServerEnd - tcpServerStart}ms`
|
||||
reportDataInput.tcpServer.output = tcpServerOutput
|
||||
reportDataInput.tcpServer.startTime = `${tcpServerEnd - tcpServerStart}ms`
|
||||
LogHelper.info(
|
||||
`TCP server startup time: ${pastebinData.tcpServer.startTime}\n`
|
||||
`TCP server startup time: ${reportDataInput.tcpServer.startTime}\n`
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -325,12 +327,12 @@ dotenv.config()
|
||||
LogHelper.error(
|
||||
`${state}. Try to generate a new one: "npm run train"\n`
|
||||
)
|
||||
pastebinData.nlpModels.globalResolversModelState = state
|
||||
reportDataInput.nlpModels.globalResolversModelState = state
|
||||
} else {
|
||||
const state = 'Found and valid'
|
||||
|
||||
LogHelper.success(`${state}\n`)
|
||||
pastebinData.nlpModels.globalResolversModelState = state
|
||||
reportDataInput.nlpModels.globalResolversModelState = state
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -354,12 +356,12 @@ dotenv.config()
|
||||
LogHelper.error(
|
||||
`${state}. Try to generate a new one: "npm run train"\n`
|
||||
)
|
||||
pastebinData.nlpModels.skillsResolversModelState = state
|
||||
reportDataInput.nlpModels.skillsResolversModelState = state
|
||||
} else {
|
||||
const state = 'Found and valid'
|
||||
|
||||
LogHelper.success(`${state}\n`)
|
||||
pastebinData.nlpModels.skillsResolversModelState = state
|
||||
reportDataInput.nlpModels.skillsResolversModelState = state
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -382,12 +384,12 @@ dotenv.config()
|
||||
LogHelper.error(
|
||||
`${state}. Try to generate a new one: "npm run train"\n`
|
||||
)
|
||||
pastebinData.nlpModels.mainModelState = state
|
||||
reportDataInput.nlpModels.mainModelState = state
|
||||
} else {
|
||||
const state = 'Found and valid'
|
||||
|
||||
LogHelper.success(`${state}\n`)
|
||||
pastebinData.nlpModels.mainModelState = state
|
||||
reportDataInput.nlpModels.mainModelState = state
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -525,10 +527,10 @@ dotenv.config()
|
||||
LogHelper.error('Please fix the errors above')
|
||||
}
|
||||
|
||||
pastebinData.report = report
|
||||
reportDataInput.report = report
|
||||
|
||||
pastebinData = JSON.parse(
|
||||
SystemHelper.sanitizeUsername(JSON.stringify(pastebinData))
|
||||
reportDataInput = JSON.parse(
|
||||
SystemHelper.sanitizeUsername(JSON.stringify(reportDataInput))
|
||||
)
|
||||
|
||||
LogHelper.title('REPORT URL')
|
||||
@@ -537,11 +539,11 @@ dotenv.config()
|
||||
|
||||
try {
|
||||
const { data } = await axios.post('https://getleon.ai/api/report', {
|
||||
report: pastebinData
|
||||
report: reportDataInput
|
||||
})
|
||||
const { data: reportData } = data
|
||||
const { data: responseReportData } = data
|
||||
|
||||
LogHelper.success(`Report URL: ${reportData.reportUrl}`)
|
||||
LogHelper.success(`Report URL: ${responseReportData.reportUrl}`)
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to send report: ${e}`)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import { LogHelper } from '@/helpers/log-helper'
|
||||
'utf8'
|
||||
)
|
||||
const regex =
|
||||
'(build|BREAKING|chore|ci|docs|feat|fix|perf|refactor|style|test)(\\((web app|docker|server|hotword|tcp server|python bridge|skill\\/([\\w-]+)))?\\)?: .{1,50}'
|
||||
'(build|BREAKING|chore|ci|docs|feat|fix|perf|refactor|style|test)(\\((web app|scripts|docker|server|hotword|tcp server|bridge\\/(python|nodejs)|skill\\/([\\w-]+)))?\\)?: .{1,50}'
|
||||
|
||||
if (commitMessage.match(regex) !== null) {
|
||||
LogHelper.success('Commit message validated')
|
||||
|
||||
@@ -14,7 +14,7 @@ dotenv.config()
|
||||
* Generate HTTP API key script
|
||||
* save it in the .env file
|
||||
*/
|
||||
const generateHttpApiKey = () =>
|
||||
const generateHTTPAPIKey = () =>
|
||||
new Promise(async (resolve, reject) => {
|
||||
LogHelper.info('Generating the HTTP API key...')
|
||||
|
||||
@@ -56,17 +56,17 @@ export default () =>
|
||||
!process.env.LEON_HTTP_API_KEY ||
|
||||
process.env.LEON_HTTP_API_KEY === ''
|
||||
) {
|
||||
await generateHttpApiKey()
|
||||
await generateHTTPAPIKey()
|
||||
} else if (!process.env.IS_DOCKER) {
|
||||
const answer = await prompt({
|
||||
type: 'confirm',
|
||||
name: 'generate.httpApiKey',
|
||||
name: 'generate.httpAPIKey',
|
||||
message: 'Do you want to regenerate the HTTP API key?',
|
||||
default: false
|
||||
})
|
||||
|
||||
if (answer.generate.httpApiKey === true) {
|
||||
await generateHttpApiKey()
|
||||
if (answer.generate.httpAPIKey === true) {
|
||||
await generateHTTPAPIKey()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,9 +25,12 @@ import {
|
||||
*/
|
||||
export const generateSchemas = async (categoryName, schemas) => {
|
||||
const categorySchemasPath = path.join(process.cwd(), 'schemas', categoryName)
|
||||
|
||||
await fs.promises.mkdir(categorySchemasPath, { recursive: true })
|
||||
|
||||
for (const [schemaName, schemaObject] of schemas.entries()) {
|
||||
const schemaPath = path.join(categorySchemasPath, `${schemaName}.json`)
|
||||
|
||||
await fs.promises.writeFile(
|
||||
schemaPath,
|
||||
JSON.stringify(
|
||||
@@ -44,6 +47,7 @@ export const generateSchemas = async (categoryName, schemas) => {
|
||||
|
||||
export default async () => {
|
||||
LogHelper.info('Generating the JSON schemas...')
|
||||
|
||||
await Promise.all([
|
||||
generateSchemas(
|
||||
'global-data',
|
||||
@@ -71,5 +75,6 @@ export default async () => {
|
||||
])
|
||||
)
|
||||
])
|
||||
|
||||
LogHelper.success('JSON schemas generated')
|
||||
}
|
||||
|
||||
@@ -3,7 +3,11 @@ import path from 'node:path'
|
||||
import { prompt } from 'inquirer'
|
||||
import { command } from 'execa'
|
||||
|
||||
import { PYTHON_BRIDGE_SRC_PATH, TCP_SERVER_SRC_PATH } from '@/constants'
|
||||
import {
|
||||
NODEJS_BRIDGE_SRC_PATH,
|
||||
PYTHON_BRIDGE_SRC_PATH,
|
||||
TCP_SERVER_SRC_PATH
|
||||
} from '@/constants'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { LoaderHelper } from '@/helpers/loader-helper'
|
||||
|
||||
@@ -15,6 +19,10 @@ import { LoaderHelper } from '@/helpers/loader-helper'
|
||||
|
||||
const BUILD_TARGETS = new Map()
|
||||
|
||||
BUILD_TARGETS.set('nodejs-bridge', {
|
||||
workflowFileName: 'pre-release-nodejs-bridge.yml',
|
||||
versionFilePath: path.join(NODEJS_BRIDGE_SRC_PATH, 'version.ts')
|
||||
})
|
||||
BUILD_TARGETS.set('python-bridge', {
|
||||
workflowFileName: 'pre-release-python-bridge.yml',
|
||||
versionFilePath: path.join(PYTHON_BRIDGE_SRC_PATH, 'version.py')
|
||||
|
||||
@@ -10,7 +10,8 @@ export default (version) =>
|
||||
LogHelper.info('Updating version...')
|
||||
|
||||
const promises = []
|
||||
const files = ['package.json', 'package-lock.json']
|
||||
// const files = ['package.json', 'package-lock.json']
|
||||
const files = ['package.json']
|
||||
|
||||
for (let i = 0; i < files.length; i += 1) {
|
||||
promises.push(
|
||||
|
||||
@@ -20,9 +20,11 @@ export default async () => {
|
||||
2
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
LogHelper.success(`Instance ID created: ${instanceID}`)
|
||||
LogHelper.success(`Instance ID created: ${instanceID}`)
|
||||
} else {
|
||||
LogHelper.success(`Instance ID already exists: ${instanceID}`)
|
||||
}
|
||||
} catch (e) {
|
||||
LogHelper.warning(`Failed to create the instance ID: ${e}`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import stream from 'node:stream'
|
||||
import readline from 'node:readline'
|
||||
|
||||
import axios from 'axios'
|
||||
import prettyBytes from 'pretty-bytes'
|
||||
import prettyMilliseconds from 'pretty-ms'
|
||||
import extractZip from 'extract-zip'
|
||||
|
||||
import {
|
||||
BINARIES_FOLDER_NAME,
|
||||
GITHUB_URL,
|
||||
NODEJS_BRIDGE_DIST_PATH,
|
||||
PYTHON_BRIDGE_DIST_PATH,
|
||||
TCP_SERVER_DIST_PATH,
|
||||
NODEJS_BRIDGE_BIN_NAME,
|
||||
PYTHON_BRIDGE_BIN_NAME,
|
||||
TCP_SERVER_BIN_NAME,
|
||||
NODEJS_BRIDGE_VERSION,
|
||||
PYTHON_BRIDGE_VERSION,
|
||||
TCP_SERVER_VERSION
|
||||
} from '@/constants'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
|
||||
/**
|
||||
* Set up binaries according to the given setup target
|
||||
* 1. Delete the existing dist binaries if already exist
|
||||
* 2. Download the latest binaries from GitHub releases
|
||||
* 3. Extract the downloaded ZIP file to the dist folder
|
||||
*/
|
||||
|
||||
const TARGETS = new Map()
|
||||
|
||||
TARGETS.set('nodejs-bridge', {
|
||||
name: 'Node.js bridge',
|
||||
distPath: NODEJS_BRIDGE_DIST_PATH,
|
||||
manifestPath: path.join(NODEJS_BRIDGE_DIST_PATH, 'manifest.json'),
|
||||
archiveName: `${NODEJS_BRIDGE_BIN_NAME.split('.')[0]}.zip`,
|
||||
version: NODEJS_BRIDGE_VERSION,
|
||||
isPlatformDependent: false // Need to be built for the target platform or not
|
||||
})
|
||||
TARGETS.set('python-bridge', {
|
||||
name: 'Python bridge',
|
||||
distPath: PYTHON_BRIDGE_DIST_PATH,
|
||||
manifestPath: path.join(PYTHON_BRIDGE_DIST_PATH, 'manifest.json'),
|
||||
archiveName: `${PYTHON_BRIDGE_BIN_NAME}-${BINARIES_FOLDER_NAME}.zip`,
|
||||
version: PYTHON_BRIDGE_VERSION,
|
||||
isPlatformDependent: true
|
||||
})
|
||||
TARGETS.set('tcp-server', {
|
||||
name: 'TCP server',
|
||||
distPath: TCP_SERVER_DIST_PATH,
|
||||
manifestPath: path.join(TCP_SERVER_DIST_PATH, 'manifest.json'),
|
||||
archiveName: `${TCP_SERVER_BIN_NAME}-${BINARIES_FOLDER_NAME}.zip`,
|
||||
version: TCP_SERVER_VERSION,
|
||||
isPlatformDependent: true
|
||||
})
|
||||
|
||||
async function createManifestFile(manifestPath, name, version) {
|
||||
const manifest = {
|
||||
name,
|
||||
version,
|
||||
setupDate: Date.now()
|
||||
}
|
||||
|
||||
await fs.promises.writeFile(manifestPath, JSON.stringify(manifest, null, 2))
|
||||
}
|
||||
|
||||
const setupBinaries = async (key) => {
|
||||
const {
|
||||
name,
|
||||
distPath,
|
||||
archiveName,
|
||||
version,
|
||||
manifestPath,
|
||||
isPlatformDependent
|
||||
} = TARGETS.get(key)
|
||||
let manifest = null
|
||||
|
||||
if (fs.existsSync(manifestPath)) {
|
||||
manifest = JSON.parse(await fs.promises.readFile(manifestPath, 'utf8'))
|
||||
|
||||
LogHelper.info(`Found ${name} ${manifest.version}`)
|
||||
LogHelper.info(`Latest version is ${version}`)
|
||||
}
|
||||
|
||||
if (!manifest || manifest.version !== version) {
|
||||
const buildPath = isPlatformDependent
|
||||
? path.join(distPath, BINARIES_FOLDER_NAME)
|
||||
: path.join(distPath, 'bin')
|
||||
const archivePath = path.join(distPath, archiveName)
|
||||
|
||||
await Promise.all([
|
||||
fs.promises.rm(buildPath, { recursive: true, force: true }),
|
||||
fs.promises.rm(archivePath, { recursive: true, force: true })
|
||||
])
|
||||
|
||||
try {
|
||||
LogHelper.info(`Downloading ${name}...`)
|
||||
|
||||
const archiveWriter = fs.createWriteStream(archivePath)
|
||||
const latestReleaseAssetURL = `${GITHUB_URL}/releases/download/${key}_v${version}/${archiveName}`
|
||||
const { data } = await axios.get(latestReleaseAssetURL, {
|
||||
responseType: 'stream',
|
||||
onDownloadProgress: ({ loaded, total, progress, estimated, rate }) => {
|
||||
const percentage = Math.floor(progress * 100)
|
||||
const downloadedSize = prettyBytes(loaded)
|
||||
const totalSize = prettyBytes(total)
|
||||
const estimatedTime = !estimated
|
||||
? 0
|
||||
: prettyMilliseconds(estimated * 1_000, { secondsDecimalDigits: 0 })
|
||||
const downloadRate = !rate ? 0 : prettyBytes(rate)
|
||||
|
||||
readline.clearLine(process.stdout, 0)
|
||||
readline.cursorTo(process.stdout, 0, null)
|
||||
process.stdout.write(
|
||||
`Download progress: ${percentage}% (${downloadedSize}/${totalSize} | ${downloadRate}/s | ${estimatedTime} ETA)`
|
||||
)
|
||||
|
||||
if (percentage === 100) {
|
||||
process.stdout.write('\n')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
data.pipe(archiveWriter)
|
||||
await stream.promises.finished(archiveWriter)
|
||||
|
||||
LogHelper.success(`${name} downloaded`)
|
||||
LogHelper.info(`Extracting ${name}...`)
|
||||
|
||||
const absoluteDistPath = isPlatformDependent
|
||||
? path.resolve(distPath)
|
||||
: path.resolve(distPath, 'bin')
|
||||
await extractZip(archivePath, { dir: absoluteDistPath })
|
||||
|
||||
LogHelper.success(`${name} extracted`)
|
||||
|
||||
await Promise.all([
|
||||
fs.promises.rm(archivePath, { recursive: true, force: true }),
|
||||
createManifestFile(manifestPath, name, version)
|
||||
])
|
||||
|
||||
LogHelper.success(`${name} manifest file created`)
|
||||
LogHelper.success(`${name} ${version} ready`)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to set up ${name}: ${error}`)
|
||||
}
|
||||
} else {
|
||||
LogHelper.success(`${name} is already at the latest version (${version})`)
|
||||
}
|
||||
}
|
||||
|
||||
export default async () => {
|
||||
await setupBinaries('nodejs-bridge')
|
||||
await setupBinaries('python-bridge')
|
||||
await setupBinaries('tcp-server')
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import stream from 'node:stream'
|
||||
import readline from 'node:readline'
|
||||
|
||||
import axios from 'axios'
|
||||
import prettyBytes from 'pretty-bytes'
|
||||
import prettyMilliseconds from 'pretty-ms'
|
||||
import extractZip from 'extract-zip'
|
||||
|
||||
import {
|
||||
BINARIES_FOLDER_NAME,
|
||||
GITHUB_URL,
|
||||
PYTHON_BRIDGE_DIST_PATH,
|
||||
TCP_SERVER_DIST_PATH,
|
||||
PYTHON_BRIDGE_BIN_NAME,
|
||||
TCP_SERVER_BIN_NAME,
|
||||
PYTHON_BRIDGE_VERSION,
|
||||
TCP_SERVER_VERSION
|
||||
} from '@/constants'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
|
||||
/**
|
||||
* Set up Python binaries according to the given setup target
|
||||
* 1. Delete the existing dist binaries if already exist
|
||||
* 2. Download the latest Python binaries from GitHub releases
|
||||
* 3. Extract the downloaded ZIP file to the dist folder
|
||||
*/
|
||||
|
||||
const PYTHON_TARGETS = new Map()
|
||||
|
||||
PYTHON_TARGETS.set('python-bridge', {
|
||||
name: 'Python bridge',
|
||||
distPath: PYTHON_BRIDGE_DIST_PATH,
|
||||
archiveName: `${PYTHON_BRIDGE_BIN_NAME}-${BINARIES_FOLDER_NAME}.zip`,
|
||||
version: PYTHON_BRIDGE_VERSION
|
||||
})
|
||||
PYTHON_TARGETS.set('tcp-server', {
|
||||
name: 'TCP server',
|
||||
distPath: TCP_SERVER_DIST_PATH,
|
||||
archiveName: `${TCP_SERVER_BIN_NAME}-${BINARIES_FOLDER_NAME}.zip`,
|
||||
version: TCP_SERVER_VERSION
|
||||
})
|
||||
|
||||
const setupPythonBinaries = async (key) => {
|
||||
const { name, distPath, archiveName, version } = PYTHON_TARGETS.get(key)
|
||||
const buildPath = path.join(distPath, BINARIES_FOLDER_NAME)
|
||||
const archivePath = path.join(distPath, archiveName)
|
||||
|
||||
await Promise.all([
|
||||
fs.promises.rm(buildPath, { recursive: true, force: true }),
|
||||
fs.promises.rm(archivePath, { recursive: true, force: true })
|
||||
])
|
||||
|
||||
try {
|
||||
LogHelper.info(`Downloading ${name}...`)
|
||||
|
||||
const archiveWriter = fs.createWriteStream(archivePath)
|
||||
const latestReleaseAssetURL = `${GITHUB_URL}/releases/download/${key}_v${version}/${archiveName}`
|
||||
const { data } = await axios.get(latestReleaseAssetURL, {
|
||||
responseType: 'stream',
|
||||
onDownloadProgress: ({ loaded, total, progress, estimated, rate }) => {
|
||||
const percentage = Math.floor(progress * 100)
|
||||
const downloadedSize = prettyBytes(loaded)
|
||||
const totalSize = prettyBytes(total)
|
||||
const estimatedTime = !estimated
|
||||
? 0
|
||||
: prettyMilliseconds(estimated * 1_000, { secondsDecimalDigits: 0 })
|
||||
const downloadRate = !rate ? 0 : prettyBytes(rate)
|
||||
|
||||
readline.clearLine(process.stdout, 0)
|
||||
readline.cursorTo(process.stdout, 0, null)
|
||||
process.stdout.write(
|
||||
`Download progress: ${percentage}% (${downloadedSize}/${totalSize} | ${downloadRate}/s | ${estimatedTime} ETA)`
|
||||
)
|
||||
|
||||
if (percentage === 100) {
|
||||
process.stdout.write('\n')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
data.pipe(archiveWriter)
|
||||
await stream.promises.finished(archiveWriter)
|
||||
|
||||
LogHelper.success(`${name} downloaded`)
|
||||
LogHelper.info(`Extracting ${name}...`)
|
||||
|
||||
const absoluteDistPath = path.resolve(distPath)
|
||||
await extractZip(archivePath, { dir: absoluteDistPath })
|
||||
|
||||
LogHelper.success(`${name} extracted`)
|
||||
|
||||
await fs.promises.rm(archivePath, { recursive: true, force: true })
|
||||
|
||||
LogHelper.success(`${name} ready`)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to set up ${name}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
export default async () => {
|
||||
await setupPythonBinaries('python-bridge')
|
||||
await setupPythonBinaries('tcp-server')
|
||||
}
|
||||
@@ -2,13 +2,13 @@ import { LoaderHelper } from '@/helpers/loader-helper'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
|
||||
import train from '../train/train'
|
||||
import generateHttpApiKey from '../generate/generate-http-api-key'
|
||||
import generateJsonSchemas from '../generate/generate-json-schemas'
|
||||
import generateHTTPAPIKey from '../generate/generate-http-api-key'
|
||||
import generateJSONSchemas from '../generate/generate-json-schemas'
|
||||
|
||||
import setupDotenv from './setup-dotenv'
|
||||
import setupCore from './setup-core'
|
||||
import setupSkillsConfig from './setup-skills-config'
|
||||
import setupPythonBinaries from './setup-python-binaries'
|
||||
import setupBinaries from './setup-binaries'
|
||||
import createInstanceID from './create-instance-id'
|
||||
|
||||
// Do not load ".env" file because it is not created yet
|
||||
@@ -22,9 +22,9 @@ import createInstanceID from './create-instance-id'
|
||||
LoaderHelper.start()
|
||||
await Promise.all([setupCore(), setupSkillsConfig()])
|
||||
LoaderHelper.stop()
|
||||
await setupPythonBinaries()
|
||||
await generateHttpApiKey()
|
||||
await generateJsonSchemas()
|
||||
await setupBinaries()
|
||||
await generateHTTPAPIKey()
|
||||
await generateJSONSchemas()
|
||||
LoaderHelper.start()
|
||||
await train()
|
||||
await createInstanceID()
|
||||
|
||||
+30
-4
@@ -18,12 +18,28 @@ export const GITHUB_URL = 'https://github.com/leon-ai/leon'
|
||||
* Binaries / distribution
|
||||
*/
|
||||
export const BINARIES_FOLDER_NAME = SystemHelper.getBinariesFolderName()
|
||||
export const PYTHON_BRIDGE_DIST_PATH = path.join('bridges', 'python', 'dist')
|
||||
export const TCP_SERVER_DIST_PATH = path.join('tcp_server', 'dist')
|
||||
export const NODEJS_BRIDGE_ROOT_PATH = path.join('bridges', 'nodejs')
|
||||
export const PYTHON_BRIDGE_ROOT_PATH = path.join('bridges', 'python')
|
||||
export const TCP_SERVER_ROOT_PATH = path.join('tcp_server')
|
||||
|
||||
export const PYTHON_BRIDGE_SRC_PATH = path.join('bridges', 'python', 'src')
|
||||
export const TCP_SERVER_SRC_PATH = path.join('tcp_server', 'src')
|
||||
export const NODEJS_BRIDGE_DIST_PATH = path.join(
|
||||
NODEJS_BRIDGE_ROOT_PATH,
|
||||
'dist'
|
||||
)
|
||||
export const PYTHON_BRIDGE_DIST_PATH = path.join(
|
||||
PYTHON_BRIDGE_ROOT_PATH,
|
||||
'dist'
|
||||
)
|
||||
export const TCP_SERVER_DIST_PATH = path.join(TCP_SERVER_ROOT_PATH, 'dist')
|
||||
|
||||
export const NODEJS_BRIDGE_SRC_PATH = path.join(NODEJS_BRIDGE_ROOT_PATH, 'src')
|
||||
export const PYTHON_BRIDGE_SRC_PATH = path.join(PYTHON_BRIDGE_ROOT_PATH, 'src')
|
||||
export const TCP_SERVER_SRC_PATH = path.join(TCP_SERVER_ROOT_PATH, 'src')
|
||||
|
||||
const NODEJS_BRIDGE_VERSION_FILE_PATH = path.join(
|
||||
NODEJS_BRIDGE_SRC_PATH,
|
||||
'version.ts'
|
||||
)
|
||||
const PYTHON_BRIDGE_VERSION_FILE_PATH = path.join(
|
||||
PYTHON_BRIDGE_SRC_PATH,
|
||||
'version.py'
|
||||
@@ -32,6 +48,9 @@ const TCP_SERVER_VERSION_FILE_PATH = path.join(
|
||||
TCP_SERVER_SRC_PATH,
|
||||
'version.py'
|
||||
)
|
||||
export const [, NODEJS_BRIDGE_VERSION] = fs
|
||||
.readFileSync(NODEJS_BRIDGE_VERSION_FILE_PATH, 'utf8')
|
||||
.split("'")
|
||||
export const [, PYTHON_BRIDGE_VERSION] = fs
|
||||
.readFileSync(PYTHON_BRIDGE_VERSION_FILE_PATH, 'utf8')
|
||||
.split("'")
|
||||
@@ -39,6 +58,7 @@ export const [, TCP_SERVER_VERSION] = fs
|
||||
.readFileSync(TCP_SERVER_VERSION_FILE_PATH, 'utf8')
|
||||
.split("'")
|
||||
|
||||
export const NODEJS_BRIDGE_BIN_NAME = 'leon-nodejs-bridge.js'
|
||||
export const PYTHON_BRIDGE_BIN_NAME = 'leon-python-bridge'
|
||||
export const TCP_SERVER_BIN_NAME = 'leon-tcp-server'
|
||||
|
||||
@@ -52,6 +72,11 @@ export const PYTHON_BRIDGE_BIN_PATH = path.join(
|
||||
BINARIES_FOLDER_NAME,
|
||||
PYTHON_BRIDGE_BIN_NAME
|
||||
)
|
||||
export const NODEJS_BRIDGE_BIN_PATH = `${process.execPath} ${path.join(
|
||||
NODEJS_BRIDGE_DIST_PATH,
|
||||
'bin',
|
||||
NODEJS_BRIDGE_BIN_NAME
|
||||
)}`
|
||||
|
||||
export const LEON_VERSION = process.env['npm_package_version']
|
||||
|
||||
@@ -115,6 +140,7 @@ export const LEON_FILE_PATH = path.join('leon.json')
|
||||
/**
|
||||
* Misc
|
||||
*/
|
||||
export const MINIMUM_REQUIRED_RAM = 4
|
||||
export const INSTANCE_ID = fs.existsSync(LEON_FILE_PATH)
|
||||
? JSON.parse(fs.readFileSync(LEON_FILE_PATH, 'utf8')).instanceID
|
||||
: null
|
||||
|
||||
@@ -9,15 +9,24 @@ import type {
|
||||
NERCustomEntity,
|
||||
NLUResult
|
||||
} from '@/core/nlp/types'
|
||||
import type { SkillConfigSchema } from '@/schemas/skill-schemas'
|
||||
import type { SkillConfigSchema, SkillSchema } from '@/schemas/skill-schemas'
|
||||
import type {
|
||||
BrainProcessResult,
|
||||
IntentObject,
|
||||
SkillResult
|
||||
} from '@/core/brain/types'
|
||||
import { SkillActionType, SkillOutputType } from '@/core/brain/types'
|
||||
import {
|
||||
SkillActionTypes,
|
||||
SkillBridges,
|
||||
SkillOutputTypes
|
||||
} from '@/core/brain/types'
|
||||
import { langs } from '@@/core/langs.json'
|
||||
import { HAS_TTS, PYTHON_BRIDGE_BIN_PATH, TMP_PATH } from '@/constants'
|
||||
import {
|
||||
HAS_TTS,
|
||||
PYTHON_BRIDGE_BIN_PATH,
|
||||
NODEJS_BRIDGE_BIN_PATH,
|
||||
TMP_PATH
|
||||
} from '@/constants'
|
||||
import { SOCKET_SERVER, TTS } from '@/core'
|
||||
import { LangHelper } from '@/helpers/lang-helper'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
@@ -162,11 +171,11 @@ export default class Brain {
|
||||
*/
|
||||
private createIntentObject(
|
||||
nluResult: NLUResult,
|
||||
utteranceId: string,
|
||||
utteranceID: string,
|
||||
slots: IntentObject['slots']
|
||||
): IntentObject {
|
||||
return {
|
||||
id: utteranceId,
|
||||
id: utteranceID,
|
||||
lang: this._lang,
|
||||
domain: nluResult.classification.domain,
|
||||
skill: nluResult.classification.skill,
|
||||
@@ -190,7 +199,7 @@ export default class Brain {
|
||||
const obj = JSON.parse(data.toString())
|
||||
|
||||
if (typeof obj === 'object') {
|
||||
if (obj.output.type === SkillOutputType.Intermediate) {
|
||||
if (obj.output.type === SkillOutputTypes.Intermediate) {
|
||||
LogHelper.title(`${this.skillFriendlyName} skill`)
|
||||
LogHelper.info(data.toString())
|
||||
|
||||
@@ -258,7 +267,8 @@ export default class Brain {
|
||||
*/
|
||||
private async executeLogicActionSkill(
|
||||
nluResult: NLUResult,
|
||||
utteranceId: string,
|
||||
skillBridge: SkillSchema['bridge'],
|
||||
utteranceID: string,
|
||||
intentObjectPath: string
|
||||
): Promise<void> {
|
||||
// Ensure the process is empty (to be able to execute other processes outside of Brain)
|
||||
@@ -273,7 +283,7 @@ export default class Brain {
|
||||
|
||||
const intentObject = this.createIntentObject(
|
||||
nluResult,
|
||||
utteranceId,
|
||||
utteranceID,
|
||||
slots
|
||||
)
|
||||
|
||||
@@ -282,10 +292,20 @@ export default class Brain {
|
||||
intentObjectPath,
|
||||
JSON.stringify(intentObject)
|
||||
)
|
||||
this.skillProcess = spawn(
|
||||
`${PYTHON_BRIDGE_BIN_PATH} "${intentObjectPath}"`,
|
||||
{ shell: true }
|
||||
)
|
||||
|
||||
if (skillBridge === SkillBridges.Python) {
|
||||
this.skillProcess = spawn(
|
||||
`${PYTHON_BRIDGE_BIN_PATH} "${intentObjectPath}"`,
|
||||
{ shell: true }
|
||||
)
|
||||
} else if (skillBridge === SkillBridges.NodeJS) {
|
||||
this.skillProcess = spawn(
|
||||
`${NODEJS_BRIDGE_BIN_PATH} "${intentObjectPath}"`,
|
||||
{ shell: true }
|
||||
)
|
||||
} else {
|
||||
LogHelper.error(`The skill bridge is not supported: ${skillBridge}`)
|
||||
}
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to save intent object: ${e}`)
|
||||
}
|
||||
@@ -299,8 +319,8 @@ export default class Brain {
|
||||
const executionTimeStart = Date.now()
|
||||
|
||||
return new Promise(async (resolve) => {
|
||||
const utteranceId = `${Date.now()}-${StringHelper.random(4)}`
|
||||
const intentObjectPath = path.join(TMP_PATH, `${utteranceId}.json`)
|
||||
const utteranceID = `${Date.now()}-${StringHelper.random(4)}`
|
||||
const intentObjectPath = path.join(TMP_PATH, `${utteranceID}.json`)
|
||||
const speeches: string[] = []
|
||||
|
||||
// Reset skill output
|
||||
@@ -334,24 +354,25 @@ export default class Brain {
|
||||
? actions[action.next_action]
|
||||
: null
|
||||
|
||||
if (actionType === SkillActionType.Logic) {
|
||||
if (actionType === SkillActionTypes.Logic) {
|
||||
/**
|
||||
* "Logic" action skill execution
|
||||
*/
|
||||
|
||||
await this.executeLogicActionSkill(
|
||||
nluResult,
|
||||
utteranceId,
|
||||
intentObjectPath
|
||||
)
|
||||
|
||||
const domainName = nluResult.classification.domain
|
||||
const skillName = nluResult.classification.skill
|
||||
const { name: domainFriendlyName } =
|
||||
await SkillDomainHelper.getSkillDomainInfo(domainName)
|
||||
const { name: skillFriendlyName } =
|
||||
const { name: skillFriendlyName, bridge: skillBridge } =
|
||||
await SkillDomainHelper.getSkillInfo(domainName, skillName)
|
||||
|
||||
await this.executeLogicActionSkill(
|
||||
nluResult,
|
||||
skillBridge,
|
||||
utteranceID,
|
||||
intentObjectPath
|
||||
)
|
||||
|
||||
this.domainFriendlyName = domainFriendlyName
|
||||
this.skillFriendlyName = skillFriendlyName
|
||||
|
||||
@@ -387,7 +408,7 @@ export default class Brain {
|
||||
|
||||
// Synchronize the downloaded content if enabled
|
||||
if (
|
||||
skillResult.output.type === SkillOutputType.End &&
|
||||
skillResult.output.type === SkillOutputTypes.End &&
|
||||
skillResult.output.options['synchronization'] &&
|
||||
skillResult.output.options['synchronization'].enabled &&
|
||||
skillResult.output.options['synchronization'].enabled ===
|
||||
@@ -442,7 +463,7 @@ export default class Brain {
|
||||
}
|
||||
|
||||
resolve({
|
||||
utteranceId,
|
||||
utteranceID,
|
||||
lang: this._lang,
|
||||
...nluResult,
|
||||
speeches,
|
||||
@@ -571,7 +592,7 @@ export default class Brain {
|
||||
}
|
||||
|
||||
resolve({
|
||||
utteranceId,
|
||||
utteranceID,
|
||||
lang: this._lang,
|
||||
...nluResult,
|
||||
speeches: [answer as string],
|
||||
|
||||
@@ -28,7 +28,7 @@ export interface SkillResult {
|
||||
entities: NEREntity[]
|
||||
slots: NLUSlots
|
||||
output: {
|
||||
type: SkillOutputType
|
||||
type: SkillOutputTypes
|
||||
codes: string[]
|
||||
speech: string
|
||||
core: SkillCoreData | undefined
|
||||
@@ -37,11 +37,15 @@ export interface SkillResult {
|
||||
}
|
||||
}
|
||||
|
||||
export enum SkillOutputType {
|
||||
export enum SkillBridges {
|
||||
Python = 'python',
|
||||
NodeJS = 'nodejs'
|
||||
}
|
||||
export enum SkillOutputTypes {
|
||||
Intermediate = 'inter',
|
||||
End = 'end'
|
||||
}
|
||||
export enum SkillActionType {
|
||||
export enum SkillActionTypes {
|
||||
Logic = 'logic',
|
||||
Dialog = 'dialog'
|
||||
}
|
||||
@@ -63,7 +67,7 @@ export interface IntentObject {
|
||||
export interface BrainProcessResult extends NLUResult {
|
||||
speeches: string[]
|
||||
executionTime: number
|
||||
utteranceId?: string
|
||||
utteranceID?: string
|
||||
lang?: ShortLanguageCode
|
||||
core?: SkillCoreData | undefined
|
||||
action?: SkillConfigSchema['actions'][string]
|
||||
|
||||
@@ -121,10 +121,10 @@ export class SystemHelper {
|
||||
|
||||
/**
|
||||
* Get the Node.js version of the current process
|
||||
* @example getNodeJSVersion() // 'v18.15.0'
|
||||
* @example getNodeJSVersion() // '18.15.0'
|
||||
*/
|
||||
public static getNodeJSVersion(): string {
|
||||
return process.version || '0.0.0'
|
||||
return process.versions.node || '0.0.0'
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+27
-2
@@ -29,8 +29,13 @@ import {
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { LangHelper } from '@/helpers/lang-helper'
|
||||
import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
|
||||
import { VOICE_CONFIG_PATH, GLOBAL_DATA_PATH } from '@/constants'
|
||||
import {
|
||||
MINIMUM_REQUIRED_RAM,
|
||||
VOICE_CONFIG_PATH,
|
||||
GLOBAL_DATA_PATH
|
||||
} from '@/constants'
|
||||
import { getGlobalEntitiesPath, getGlobalResolversPath } from '@/utilities'
|
||||
import { SystemHelper } from '@/helpers/system-helper'
|
||||
|
||||
interface ObjectUnknown {
|
||||
[key: string]: unknown
|
||||
@@ -66,8 +71,11 @@ const validateSchema = (
|
||||
|
||||
/**
|
||||
* Pre-checking
|
||||
* Ensure JSON files are correctly formatted
|
||||
*
|
||||
* - Ensure the system requirements are met
|
||||
* - Ensure JSON files are correctly formatted
|
||||
*/
|
||||
|
||||
const VOICE_CONFIG_SCHEMAS = {
|
||||
amazon: amazonVoiceConfiguration,
|
||||
'google-cloud': googleCloudVoiceConfiguration,
|
||||
@@ -83,6 +91,23 @@ const GLOBAL_DATA_SCHEMAS = {
|
||||
;(async (): Promise<void> => {
|
||||
LogHelper.title('Pre-checking')
|
||||
|
||||
/**
|
||||
* System requirements checking
|
||||
*/
|
||||
LogHelper.info('Checking system requirements...')
|
||||
|
||||
const totalRAMInGB = Math.round(SystemHelper.getTotalRAM())
|
||||
|
||||
if (totalRAMInGB < MINIMUM_REQUIRED_RAM) {
|
||||
LogHelper.warning(
|
||||
`Total RAM: ${totalRAMInGB} GB. Leon needs at least ${MINIMUM_REQUIRED_RAM} GB of RAM. It may not work as expected.`
|
||||
)
|
||||
} else {
|
||||
LogHelper.success(
|
||||
`Minimum required RAM: ${MINIMUM_REQUIRED_RAM} GB | Total RAM: ${totalRAMInGB} GB`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice configuration checking
|
||||
*/
|
||||
|
||||
@@ -2,8 +2,13 @@ import type { Static } from '@sinclair/typebox'
|
||||
import { Type } from '@sinclair/typebox'
|
||||
|
||||
import { globalResolverSchemaObject } from '@/schemas/global-data-schemas'
|
||||
import { SkillBridges } from '@/core/brain/types'
|
||||
|
||||
const skillBridges = [Type.Literal('python'), Type.Null()]
|
||||
const skillBridges = [
|
||||
Type.Literal(SkillBridges.Python),
|
||||
Type.Literal(SkillBridges.NodeJS),
|
||||
Type.Null()
|
||||
]
|
||||
const skillActionTypes = [
|
||||
Type.Literal('logic', {
|
||||
description: 'It runs the business logic implemented in actions via code.'
|
||||
|
||||
+25
-10
@@ -19,6 +19,7 @@ import {
|
||||
IS_PRODUCTION_ENV,
|
||||
LANG,
|
||||
LEON_VERSION,
|
||||
NODEJS_BRIDGE_VERSION,
|
||||
PYTHON_BRIDGE_VERSION,
|
||||
STT_PROVIDER,
|
||||
TCP_SERVER_VERSION,
|
||||
@@ -47,13 +48,17 @@ export class Telemetry {
|
||||
timeout: 7_000
|
||||
})
|
||||
|
||||
public static async postInstall(): Promise<PostIntallResponse> {
|
||||
const { data } = await this.axios.post('/on-post-install', {
|
||||
instanceID: this.instanceID,
|
||||
isGitpod: IS_GITPOD
|
||||
})
|
||||
public static async postInstall(): Promise<PostIntallResponse | unknown> {
|
||||
try {
|
||||
const { data } = await this.axios.post('/on-post-install', {
|
||||
instanceID: this.instanceID,
|
||||
isGitpod: IS_GITPOD
|
||||
})
|
||||
|
||||
return data
|
||||
return data
|
||||
} catch (e) {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
public static async start(): Promise<void> {
|
||||
@@ -68,6 +73,7 @@ export class Telemetry {
|
||||
sttProvider: STT_PROVIDER,
|
||||
ttsProvider: TTS_PROVIDER,
|
||||
coreVersion: LEON_VERSION,
|
||||
nodeJSBridgeVersion: NODEJS_BRIDGE_VERSION,
|
||||
pythonBridgeVersion: PYTHON_BRIDGE_VERSION,
|
||||
tcpServerVersion: TCP_SERVER_VERSION,
|
||||
environment: {
|
||||
@@ -93,10 +99,19 @@ export class Telemetry {
|
||||
}
|
||||
data.environment.osDetails.distro = os
|
||||
|
||||
await this.axios.post('/on-start', {
|
||||
instanceID: this.instanceID,
|
||||
data
|
||||
})
|
||||
try {
|
||||
await this.axios.post('/on-start', {
|
||||
instanceID: this.instanceID,
|
||||
data
|
||||
})
|
||||
} catch (e) {
|
||||
if (IS_DEVELOPMENT_ENV) {
|
||||
LogHelper.title('Telemetry')
|
||||
LogHelper.warning(
|
||||
`Failed to send start data to telemetry service: ${e}`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
} else {
|
||||
await this.axios.post('/on-start', {
|
||||
|
||||
Reference in New Issue
Block a user