mirrors/msgbyte--tianji
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:30:54 +08:00
commit 38e12526f7
1648 changed files with 490801 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/backup-db.sh
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yml"
if [ ! -f "$COMPOSE_FILE" ]; then
COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.yml"
fi
BACKUP_DIR="${BACKUP_DIR:-${SCRIPT_DIR}/backups}"
BACKUP_PREFIX="${POSTGRES_BACKUP_PREFIX:-tianji-postgres}"
KEEP_LOCAL_COPY="${KEEP_LOCAL_COPY:-false}"
KEEP_PARTIAL_DUMP="${KEEP_PARTIAL_DUMP:-false}"
R2_REGION="${R2_REGION:-auto}"
R2_PREFIX="${R2_PREFIX:-tianji}"
R2_ENDPOINT="${R2_ENDPOINT:-}"
log() {
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >&2
}
abort() {
log "ERROR: $*"
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || abort "Command '$1' is required but not found in PATH."
}
require_env() {
local var_name="$1"
local value="${!var_name:-}"
[ -n "$value" ] || abort "Environment variable '$var_name' is required."
}
[ -f "$COMPOSE_FILE" ] || abort "docker-compose.yml not found next to the script or one directory above."
require_command docker
DOCKER_COMPOSE_CMD="docker compose"
if ! docker compose version >/dev/null 2>&1; then
if command -v docker-compose >/dev/null 2>&1; then
DOCKER_COMPOSE_CMD="docker-compose"
else
abort "Neither 'docker compose' nor 'docker-compose' command is available."
fi
fi
if ! $DOCKER_COMPOSE_CMD -f "$COMPOSE_FILE" config --services | grep -qx 'postgres'; then
abort "Service 'postgres' not found in docker-compose.yml."
fi
POSTGRES_CONTAINER_ID="$($DOCKER_COMPOSE_CMD -f "$COMPOSE_FILE" ps -q postgres || true)"
if [ -z "$POSTGRES_CONTAINER_ID" ]; then
log "Postgres container is not running. Bringing it up..."
$DOCKER_COMPOSE_CMD -f "$COMPOSE_FILE" up -d postgres >/dev/null
POSTGRES_CONTAINER_ID="$($DOCKER_COMPOSE_CMD -f "$COMPOSE_FILE" ps -q postgres || true)"
[ -n "$POSTGRES_CONTAINER_ID" ] || abort "Failed to start postgres service."
fi
mkdir -p "$BACKUP_DIR"
TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
BACKUP_FILENAME="${BACKUP_PREFIX}_${TIMESTAMP}.dump"
LOCAL_BACKUP_PATH="${BACKUP_DIR}/${BACKUP_FILENAME}"
log "Creating Postgres dump at ${LOCAL_BACKUP_PATH}"
cleanup_partial_dump() {
if [ -f "$LOCAL_BACKUP_PATH" ] && [ "$KEEP_PARTIAL_DUMP" != "true" ]; then
rm -f "$LOCAL_BACKUP_PATH"
fi
}
trap cleanup_partial_dump EXIT
$DOCKER_COMPOSE_CMD -f "$COMPOSE_FILE" exec -T postgres bash -c 'set -euo pipefail; export PGPASSWORD="${POSTGRES_PASSWORD}"; pg_dump --format=custom --no-owner --no-acl -U "${POSTGRES_USER}" "${POSTGRES_DB:-${POSTGRES_USER}}"' >"$LOCAL_BACKUP_PATH"
trap - EXIT
require_command aws
require_env R2_BUCKET
require_env R2_ACCOUNT_ID
require_env R2_ACCESS_KEY_ID
require_env R2_SECRET_ACCESS_KEY
R2_ENDPOINT="${R2_ENDPOINT:-https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com}"
REMOTE_KEY="$BACKUP_FILENAME"
if [ -n "$R2_PREFIX" ]; then
R2_PREFIX="${R2_PREFIX%/}"
REMOTE_KEY="${R2_PREFIX}/${REMOTE_KEY}"
fi
REMOTE_URI="s3://${R2_BUCKET%/}/${REMOTE_KEY}"
log "Uploading backup to ${REMOTE_URI} via R2"
AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" \
AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY" \
AWS_DEFAULT_REGION="$R2_REGION" \
aws s3 cp "$LOCAL_BACKUP_PATH" "$REMOTE_URI" --endpoint-url "$R2_ENDPOINT"
if [ "$KEEP_LOCAL_COPY" != "true" ]; then
log "Removing local backup ${LOCAL_BACKUP_PATH}"
rm -f "$LOCAL_BACKUP_PATH"
else
log "Keeping local backup ${LOCAL_BACKUP_PATH}"
fi
log "Backup completed successfully"
scripts/build-geo.ts
+55
View File
@@ -0,0 +1,55 @@
import 'dotenv/config';
import fs from 'fs';
import path from 'path';
import https from 'https';
import zlib from 'zlib';
import tar from 'tar';
if (process.env.VERCEL) {
console.log('Vercel environment detected. Skipping geo setup.');
process.exit(0);
}
const db = 'GeoLite2-City';
let url = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`;
if (process.env.MAXMIND_LICENSE_KEY) {
url =
`https://download.maxmind.com/app/geoip_download` +
`?edition_id=${db}&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`;
}
const dest = path.resolve(process.cwd(), './geo');
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest);
}
const download = (url: string): Promise<NodeJS.WritableStream> =>
new Promise((resolve) => {
https.get(url, (res) => {
resolve(res.pipe(zlib.createGunzip({})).pipe(tar.t()));
});
});
download(url).then(
(res) =>
new Promise<void>((resolve, reject) => {
res.on('entry', (entry) => {
if (entry.path.endsWith('.mmdb')) {
const filename = path.join(dest, path.basename(entry.path));
entry.pipe(fs.createWriteStream(filename));
console.log('Saved geo database:', filename);
}
});
res.on('error', (e) => {
reject(e);
});
res.on('finish', () => {
resolve();
});
})
);
scripts/build-openapi-schema.ts
+16
View File
@@ -0,0 +1,16 @@
import { trpcOpenapiDocument } from '../src/server/trpc';
import fs from 'fs-extra';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const target = path.resolve(__dirname, '../website/static/openapi.json');
fs.writeJSON(target, trpcOpenapiDocument)
.then(() => {
console.log('openapi schema has been write into:', target);
})
.catch((err) => {
console.error(err);
});
scripts/build-tracker.ts
+40
View File
@@ -0,0 +1,40 @@
import { resolve } from 'path';
import * as vite from 'vite';
console.log('Start Build Tracker');
vite
.build({
resolve: {
alias: {
'tianji-client-sdk/tracker/core': resolve(
process.cwd(),
'./packages/client-sdk/src/tracker/core'
),
'tianji-client-sdk': resolve(
process.cwd(),
'./packages/client-sdk/src'
),
},
},
build: {
target: 'esnext',
lib: {
entry: resolve(process.cwd(), './src/tracker/index.ts'),
name: 'tianji',
fileName: () => 'tracker.js',
formats: ['iife'],
},
emptyOutDir: false,
outDir: resolve(process.cwd(), './src/client/public'),
minify: 'terser',
terserOptions: {
compress: {
drop_console: true,
},
},
},
})
.then((res) => {
console.log('Build Tracker Completed');
});
scripts/fetch-llm-model-v2.ts
+89
View File
@@ -0,0 +1,89 @@
// fetch llm model data from `https://models.dev/api.json`
// and output to `src/server/utils/model_prices_and_context_window_v2.json`
import 'dotenv/config';
import fs from 'fs-extra';
import path from 'path';
import { fileURLToPath } from 'url';
import { z } from 'zod';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const SOURCE_URL = 'https://models.dev/api.json';
const TARGET_PATH = path.resolve(
__dirname,
'../src/server/utils/model_prices_and_context_window_v2.json'
);
const modelSchema = z
.object({
id: z.string(),
name: z.string(),
cost: z
.object({
input: z.number().optional(),
output: z.number().optional(),
})
.passthrough()
.optional(),
limit: z
.object({
context: z.number().optional(),
output: z.number().optional(),
})
.passthrough()
.optional(),
})
.passthrough();
const providerSchema = z
.object({
name: z.string(),
models: z.record(z.string(), modelSchema),
})
.passthrough();
const llmModelDataSchema = z.record(z.string(), providerSchema);
export type LLMModelData = z.infer<typeof llmModelDataSchema>;
export type LLMProvider = z.infer<typeof providerSchema>;
export type LLMModel = z.infer<typeof modelSchema>;
async function fetchLLMModelDataV2() {
try {
console.log(`Fetching LLM model data from: ${SOURCE_URL}`);
const response = await fetch(SOURCE_URL);
if (!response.ok) {
throw new Error(
`Failed to fetch data: ${response.status} ${response.statusText}`
);
}
const raw = await response.json();
llmModelDataSchema.parse(raw);
const targetDir = path.dirname(TARGET_PATH);
await fs.ensureDir(targetDir);
await fs.writeJSON(TARGET_PATH, raw, { spaces: 2 });
console.log(`LLM model data v2 has been written to: ${TARGET_PATH}`);
console.log(`Total providers: ${Object.keys(raw).length}`);
let totalModels = 0;
Object.values(raw).forEach((provider: any) => {
if (provider.models) {
totalModels += Object.keys(provider.models).length;
}
});
console.log(`Total models: ${totalModels}`);
} catch (err) {
console.error('Error fetching or writing LLM model data v2:', err);
process.exit(1);
}
}
fetchLLMModelDataV2();
scripts/fetch-llm-model.ts
+47
View File
@@ -0,0 +1,47 @@
// fetch llm model price and window context from `https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json`
// and output to `src/server/utils/model_prices_and_context_window.json`
import 'dotenv/config';
import fs from 'fs-extra';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const SOURCE_URL =
'https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json';
const TARGET_PATH = path.resolve(
__dirname,
'../src/server/utils/model_prices_and_context_window.json'
);
async function fetchLLMModelData() {
try {
console.log(`Fetching LLM model data from: ${SOURCE_URL}`);
const response = await fetch(SOURCE_URL);
if (!response.ok) {
throw new Error(
`Failed to fetch data: ${response.status} ${response.statusText}`
);
}
const data = await response.json();
// Ensure target directory exists
const targetDir = path.dirname(TARGET_PATH);
await fs.ensureDir(targetDir);
// Write data to file
await fs.writeJSON(TARGET_PATH, data, { spaces: 2 });
console.log(`LLM model data has been written to: ${TARGET_PATH}`);
} catch (err) {
console.error('Error fetching or writing LLM model data:', err);
process.exit(1);
}
}
fetchLLMModelData();
scripts/install.sh
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env bash
#=================================================
# System Required: CentOS/Debian/ArchLinux with Systemd Support
# Description: Tianji Report
# Author: moonrailgun
#=================================================
Green_font_prefix="\033[32m" && Red_font_prefix="\033[31m" && Red_background_prefix="\033[41;37m" && Font_color_suffix="\033[0m"
Info="${Green_font_prefix}[Info]${Font_color_suffix}"
Error="${Red_font_prefix}[Error]${Font_color_suffix}"
Tip="${Green_font_prefix}[Tip]${Font_color_suffix}"
function check_sys() {
if [[ -f /etc/redhat-release ]]; then
release="centos"
elif grep -q -E -i "debian|ubuntu" /etc/issue; then
release="debian"
elif grep -q -E -i "centos|red hat|redhat" /etc/issue; then
release="centos"
elif grep -q -E -i "Arch|Manjaro" /etc/issue; then
release="arch"
elif grep -q -E -i "debian|ubuntu" /proc/version; then
release="debian"
elif grep -q -E -i "centos|red hat|redhat" /proc/version; then
release="centos"
else
echo -e "Tianji Report not support this linux version"
exit 1
fi
bit=$(uname -m)
}
function check_pid() {
PID=$(pgrep -f "tianji-reporter")
}
function install_dependencies() {
case ${release} in
centos)
yum install -y wget curl
;;
debian)
apt-get update -y
apt-get install -y wget curl
;;
arch)
pacman -Syu --noconfirm wget curl
;;
*)
exit 1
;;
esac
}
function input_dsn() {
defaultServer="{{DEFAULT_SERVER}}"
echo -e "${Info} Please input server url, press enter use default value(default: $defaultServer)"
read -re serverUrl
serverUrl=${serverUrl:-$defaultServer}
defaultWorkspace="{{DEFAULT_WORKSPACE}}"
echo -e "${Info} Please input workspaceId, press enter use default value(default: $defaultWorkspace)"
read -re workspace
workspace=${workspace:-$defaultWorkspace}
}
service_conf=/usr/lib/systemd/system/tianji-reporter.service
function write_service() {
echo -e "${Info} Write to systemd configuration"
cat >${service_conf} <<-EOF
[Unit]
Description=Tianji-Reporter
Documentation=https://github.com/msgbyte/tianji
After=network.target
[Service]
ExecStart=/usr/local/tianji/reporter/tianji-reporter --url "${serverUrl}" --workspace "${workspace}"
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
}
function enable_service() {
write_service
systemctl enable tianji-reporter
systemctl start tianji-reporter
check_pid
if [[ -n ${PID} ]]; then
echo -e "${Info} Tianji Reporter Started!"
else
echo -e "${Error} Tianji Reporter Start Failed!"
fi
}
function restart_service() {
write_service
systemctl daemon-reload
systemctl restart tianji-reporter
check_pid
if [[ -n ${PID} ]]; then
echo -e "${Info} Tianji Reporter Started!"
else
echo -e "${Error} Tianji Reporter Start Failed!"
fi
}
function reset_config() {
restart_service
}
function install_client() {
case ${bit} in
x86_64)
arch=amd64
;;
# i386)
# arch=386
# ;;
aarch64 | aarch64_be | arm64 | armv8b | armv8l)
arch=arm64
;;
# arm | armv6l | armv7l | armv5tel | armv5tejl)
# arch=arm
# ;;
# mips | mips64)
# arch=mips
# ;;
*)
exit 1
;;
esac
echo -e "${Info} Downloading ${arch} binary file..."
mkdir -p /usr/local/tianji/reporter/
cd /tmp
wget --no-check-certificate "https://github.com/msgbyte/tianji/releases/latest/download/tianji-reporter-linux-${arch}" -O tianji-reporter-linux-${arch}
mv tianji-reporter-linux-${arch} /usr/local/tianji/reporter/tianji-reporter
chmod +x /usr/local/tianji/reporter/tianji-reporter
enable_service
}
function auto_install() {
dsn=$(echo ${*})
install_client
}
function uninstall_client() {
systemctl stop tianji-reporter
systemctl disable tianji-reporter
rm -rf /usr/local/tianji/reporter/
rm -rf /usr/lib/systemd/system/tianji-reporter.service
}
check_sys
case "$1" in
uninstall|uni)
uninstall_client
;;
reset_conf|re)
input_dsn
reset_config
;;
-dsn)
shift 1
install_dependencies
auto_install ${*}
;;
*)
install_dependencies
input_dsn
install_client
;;
esac
scripts/run-backup-example.sh
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -euo pipefail
# Example startup script for backup-db.sh. Adjust values as needed before running.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export BACKUP_DIR="${BACKUP_DIR:-${SCRIPT_DIR}/backups}" # Directory to store temporary backup files
export POSTGRES_BACKUP_PREFIX="${POSTGRES_BACKUP_PREFIX:-tianji-postgres}" # Prefix for backup file names
export KEEP_LOCAL_COPY="${KEEP_LOCAL_COPY:-false}" # Set to "true" to keep local dump after upload
export KEEP_PARTIAL_DUMP="${KEEP_PARTIAL_DUMP:-false}" # Set to "true" to keep partial dump if script fails
# PostgreSQL container must expose these environment variables.
export POSTGRES_USER="${POSTGRES_USER:-tianji}"
export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-tianji}"
export POSTGRES_DB="${POSTGRES_DB:-tianji}"
# Cloudflare R2 credentials. Replace placeholders with real values.
export R2_BUCKET="${R2_BUCKET:-your-r2-bucket}"
export R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-your-account-id}"
export R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID:-your-access-key}"
export R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-your-secret-key}"
export R2_REGION="${R2_REGION:-auto}"
export R2_PREFIX="${R2_PREFIX:-}" # Optional path prefix in the bucket
export R2_ENDPOINT="${R2_ENDPOINT:-}" # Optional custom endpoint, defaults to Cloudflare domain in backup script
exec "${SCRIPT_DIR}/backup-db.sh"
scripts/start-tianji-container.sh
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# Start Tianji server in background
pnpm start:docker &
# Wait a moment for server to start
sleep 10
# Start reporter with default workspace
/usr/local/bin/tianji-reporter --url "http://localhost:12345" --workspace "clnzoxcy10001vy2ohi4obbi0" --name "tianji-container" --silent > /dev/null &
# Wait for any process to exit
wait -n
# Exit with status of process that exited first
exit $?