chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,50 @@
|
||||
# SSO Test Environment Configuration
|
||||
# Configures OM server to use mock OIDC provider for E2E testing.
|
||||
#
|
||||
# Usage:
|
||||
# 1. Start mock OIDC provider first:
|
||||
# docker compose --profile sso-test up -d mock-oidc-provider
|
||||
#
|
||||
# 2. Start (or restart) OM server with this env file:
|
||||
# docker compose --env-file .env.sso-test up -d
|
||||
#
|
||||
# 3. Run SSO Playwright tests:
|
||||
# cd openmetadata-ui/src/main/resources/ui
|
||||
# npx playwright test --config=playwright.sso.config.ts
|
||||
#
|
||||
# NOTE: Server-side URLs use Docker hostname (mock-oidc-provider:9090).
|
||||
# Client-side URLs (authority, callback) use localhost:9090.
|
||||
|
||||
# Authentication provider — custom-oidc uses OidcAuthenticator
|
||||
AUTHENTICATION_PROVIDER=custom-oidc
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=mock-oidc
|
||||
AUTHENTICATION_CLIENT_TYPE=confidential
|
||||
AUTHENTICATION_RESPONSE_TYPE=code
|
||||
|
||||
# Public keys — fetched server-side, must use Docker network hostname
|
||||
AUTHENTICATION_PUBLIC_KEYS=[http://mock-oidc-provider:9090/jwks,http://localhost:8585/api/v1/system/config/jwks]
|
||||
# Authority — sent to browser for redirect, must use localhost
|
||||
AUTHENTICATION_AUTHORITY=http://localhost:9090
|
||||
AUTHENTICATION_CLIENT_ID=openmetadata-test
|
||||
AUTHENTICATION_CALLBACK_URL=http://localhost:8585/callback
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP=true
|
||||
# Map principal claims: username from `sub`, email from the `email` claim.
|
||||
# Exercises the OIDC self-signup mapped-email path (issue #29189).
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING=[username:sub,email:email]
|
||||
|
||||
# OIDC Configuration — discovery URI is fetched server-side
|
||||
OIDC_CLIENT_ID=openmetadata-test
|
||||
OIDC_TYPE=custom-oidc
|
||||
OIDC_CLIENT_SECRET=openmetadata-test-secret
|
||||
OIDC_SCOPE=openid email profile
|
||||
OIDC_DISCOVERY_URI=http://mock-oidc-provider:9090/.well-known/openid-configuration
|
||||
OIDC_USE_NONCE=true
|
||||
OIDC_PREFERRED_JWS=RS256
|
||||
OIDC_RESPONSE_TYPE=code
|
||||
OIDC_DISABLE_PKCE=true
|
||||
OIDC_CALLBACK=http://localhost:8585/callback
|
||||
OIDC_SERVER_URL=http://localhost:8585
|
||||
OIDC_CLIENT_AUTH_METHOD=client_secret_post
|
||||
OIDC_SESSION_EXPIRY=604800
|
||||
AUTHENTICATION_SESSION_EXPIRY=604800
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER=5
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build stage
|
||||
FROM alpine:3 AS build
|
||||
|
||||
COPY openmetadata-dist/target/openmetadata-*.tar.gz /
|
||||
|
||||
RUN mkdir -p /opt/openmetadata && \
|
||||
tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
|
||||
rm openmetadata-*.tar.gz
|
||||
|
||||
# Final stage
|
||||
FROM alpine:3
|
||||
|
||||
EXPOSE 8585
|
||||
|
||||
RUN adduser -D openmetadata && \
|
||||
apk update && \
|
||||
apk upgrade && \
|
||||
apk add --update --no-cache bash openjdk21-jre
|
||||
|
||||
COPY --chown=openmetadata:openmetadata --from=build /opt/openmetadata /opt/openmetadata
|
||||
COPY --chmod=755 docker/openmetadata-start.sh /
|
||||
|
||||
USER openmetadata
|
||||
|
||||
WORKDIR /opt/openmetadata
|
||||
ENTRYPOINT [ "/bin/bash" ]
|
||||
CMD ["/openmetadata-start.sh"]
|
||||
@@ -0,0 +1,253 @@
|
||||
# Distributed Search Indexing Test Environment
|
||||
|
||||
This directory contains scripts and configurations to test the distributed search indexing feature with multiple OpenMetadata servers sharing a common database.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Test Environment │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ OM Server│ │ OM Server│ │ OM Server│ │
|
||||
│ │ :8585 │ │ :8587 │ │ :8589 │ │
|
||||
│ │ SERVER-1 │ │ SERVER-2 │ │ SERVER-3 │ │
|
||||
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
|
||||
│ │ │ │ │
|
||||
│ └─────────────┼─────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────┴──────┐ │
|
||||
│ │ Polling │ (DB-based coordination) │
|
||||
│ └──────┬──────┘ │
|
||||
│ │ │
|
||||
│ ┌─────────────┴─────────────┐ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌─────────┐ ┌───────────┐ │
|
||||
│ │ MySQL │ │ OpenSearch│ │
|
||||
│ │ :3306 │ │ :9200 │ │
|
||||
│ └─────────┘ └───────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Quick Start (Docker Compose)
|
||||
|
||||
### 1. Start the Environment
|
||||
|
||||
```bash
|
||||
cd docker/development/distributed-test
|
||||
|
||||
# Start all services (builds images on first run)
|
||||
./scripts/start.sh
|
||||
|
||||
# Or force rebuild
|
||||
./scripts/start.sh --build
|
||||
```
|
||||
|
||||
### 2. Load Test Data
|
||||
|
||||
```bash
|
||||
# Load 10,000 tables (default)
|
||||
./scripts/perf-test.sh
|
||||
|
||||
# Or specify the number
|
||||
./scripts/perf-test.sh --tables 50000 --databases 50
|
||||
```
|
||||
|
||||
### 3. Trigger Reindexing
|
||||
|
||||
```bash
|
||||
# Trigger reindex on server 1
|
||||
./scripts/trigger-reindex.sh
|
||||
|
||||
# With index recreation
|
||||
./scripts/trigger-reindex.sh --recreate
|
||||
|
||||
# Specific entities only
|
||||
./scripts/trigger-reindex.sh --entities table,dashboard
|
||||
```
|
||||
|
||||
### 4. Monitor Progress
|
||||
|
||||
```bash
|
||||
# Follow logs from all servers
|
||||
./scripts/logs.sh -f
|
||||
|
||||
# Filter by pattern
|
||||
./scripts/logs.sh -f --grep "partition"
|
||||
|
||||
# Single server logs
|
||||
./scripts/logs.sh -f --server 1
|
||||
```
|
||||
|
||||
### 5. Stop the Environment
|
||||
|
||||
```bash
|
||||
# Stop containers (preserve data)
|
||||
./scripts/stop.sh
|
||||
|
||||
# Stop and clean up volumes
|
||||
./scripts/stop.sh --clean
|
||||
```
|
||||
|
||||
## Local Development (IDE Debugging)
|
||||
|
||||
For debugging with breakpoints, run OM servers locally while using Docker for MySQL and OpenSearch.
|
||||
|
||||
### 1. Start Dependencies Only
|
||||
|
||||
```bash
|
||||
cd docker/development/distributed-test
|
||||
docker compose -f local/docker-compose-deps.yml up -d
|
||||
```
|
||||
|
||||
### 2. Run Migrations (First Time)
|
||||
|
||||
```bash
|
||||
cd /path/to/openmetadata
|
||||
./bootstrap/openmetadata-ops.sh -d migrate --force
|
||||
```
|
||||
|
||||
### 3. Option A: Run from Terminal
|
||||
|
||||
```bash
|
||||
# Start all 3 servers in separate terminals
|
||||
./local/run-local-servers.sh
|
||||
|
||||
# Or specific servers
|
||||
./local/run-local-servers.sh 1 2
|
||||
```
|
||||
|
||||
### 3. Option B: Run from IDE
|
||||
|
||||
Create run configurations in IntelliJ IDEA:
|
||||
|
||||
**Server 1:**
|
||||
- Main class: `org.openmetadata.service.OpenMetadataApplication`
|
||||
- Program arguments: `server docker/development/distributed-test/local/server1.yaml`
|
||||
- VM options: `-Xmx1G -Xms512M`
|
||||
- Working directory: Project root
|
||||
|
||||
**Server 2:**
|
||||
- Same as above but with `local/server2.yaml`
|
||||
|
||||
**Server 3:**
|
||||
- Same as above but with `local/server3.yaml`
|
||||
|
||||
## Server Ports
|
||||
|
||||
| Server | API Port | Admin Port |
|
||||
|--------|----------|------------|
|
||||
| Server 1 | 8585 | 8586 |
|
||||
| Server 2 | 8587 | 8588 |
|
||||
| Server 3 | 8589 | 8590 |
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `.env` to customize:
|
||||
|
||||
```bash
|
||||
# Number of tables for test data
|
||||
TEST_DATA_TABLES=10000
|
||||
|
||||
# Log level
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Heap size per server
|
||||
OPENMETADATA_HEAP_OPTS=-Xmx1G -Xms1G
|
||||
```
|
||||
|
||||
## Testing Distributed Indexing
|
||||
|
||||
### Verify Partition Distribution
|
||||
|
||||
1. Start all 3 servers
|
||||
2. Load test data: `./scripts/perf-test.sh --tables 10000`
|
||||
3. Trigger reindex: `./scripts/trigger-reindex.sh --recreate`
|
||||
4. Watch logs: `./scripts/logs.sh -f --grep "partition"`
|
||||
|
||||
You should see output like:
|
||||
```
|
||||
[SERVER-1] INFO Claimed partition: table_0-999 (1000 records)
|
||||
[SERVER-2] INFO Claimed partition: table_1000-1999 (1000 records)
|
||||
[SERVER-3] INFO Claimed partition: table_2000-2999 (1000 records)
|
||||
[SERVER-1] INFO Completed partition: table_0-999
|
||||
...
|
||||
```
|
||||
|
||||
### Test Server Failure Recovery
|
||||
|
||||
1. Start reindexing with many partitions
|
||||
2. Stop one server mid-process: `docker stop distributed_test_om_server_2`
|
||||
3. Watch remaining servers pick up orphaned partitions
|
||||
4. Verify job completes successfully
|
||||
|
||||
### Check Job Status
|
||||
|
||||
```bash
|
||||
# Via API
|
||||
curl -s http://localhost:8585/api/v1/apps/name/SearchIndexingApplication/status | jq
|
||||
|
||||
# Check partition table directly
|
||||
docker exec -it distributed_test_mysql mysql -uopenmetadata_user -popenmetadata_password openmetadata_db \
|
||||
-e "SELECT status, COUNT(*) FROM search_index_partition GROUP BY status"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Servers Not Starting
|
||||
|
||||
Check if ports are in use:
|
||||
```bash
|
||||
lsof -i :8585
|
||||
lsof -i :8587
|
||||
lsof -i :8589
|
||||
```
|
||||
|
||||
### Database Connection Issues
|
||||
|
||||
Verify MySQL is accessible:
|
||||
```bash
|
||||
docker exec -it distributed_test_mysql mysql -uopenmetadata_user -popenmetadata_password -e "SELECT 1"
|
||||
```
|
||||
|
||||
### OpenSearch Not Ready
|
||||
|
||||
Check health:
|
||||
```bash
|
||||
curl http://localhost:9200/_cluster/health?pretty
|
||||
```
|
||||
|
||||
### View Full Logs
|
||||
|
||||
```bash
|
||||
# All container logs
|
||||
docker compose logs -f
|
||||
|
||||
# Specific container
|
||||
docker logs -f distributed_test_om_server_1
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
distributed-test/
|
||||
├── docker-compose.yml # Full environment (3 OM servers + deps)
|
||||
├── .env # Configuration variables
|
||||
├── config/
|
||||
│ └── mysql-init.sql # Database initialization
|
||||
├── scripts/
|
||||
│ ├── start.sh # Start full environment
|
||||
│ ├── stop.sh # Stop environment
|
||||
│ ├── logs.sh # View aggregated logs
|
||||
│ ├── trigger-reindex.sh # Trigger reindexing
|
||||
│ └── perf-test.sh # Load test data
|
||||
├── local/
|
||||
│ ├── docker-compose-deps.yml # Dependencies only (for IDE debugging)
|
||||
│ ├── server1.yaml # Server 1 config (port 8585)
|
||||
│ ├── server2.yaml # Server 2 config (port 8587)
|
||||
│ ├── server3.yaml # Server 3 config (port 8589)
|
||||
│ └── run-local-servers.sh # Start servers locally
|
||||
└── README.md # This file
|
||||
```
|
||||
@@ -0,0 +1,13 @@
|
||||
-- MySQL initialization script for distributed test environment
|
||||
|
||||
-- Create the OpenMetadata database
|
||||
CREATE DATABASE IF NOT EXISTS openmetadata_db;
|
||||
|
||||
-- Create the OpenMetadata user
|
||||
CREATE USER IF NOT EXISTS 'openmetadata_user'@'%' IDENTIFIED BY 'openmetadata_password';
|
||||
|
||||
-- Grant privileges
|
||||
GRANT ALL PRIVILEGES ON openmetadata_db.* TO 'openmetadata_user'@'%';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openmetadata_user'@'%';
|
||||
|
||||
FLUSH PRIVILEGES;
|
||||
@@ -0,0 +1,270 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Distributed Search Indexing Test Environment
|
||||
# This compose file sets up multiple OM servers sharing a common MySQL and OpenSearch
|
||||
|
||||
version: "3.9"
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
distributed-test-net:
|
||||
name: distributed_test_network
|
||||
driver: bridge
|
||||
|
||||
services:
|
||||
# Shared MySQL Database
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: distributed_test_mysql
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password}
|
||||
command: >
|
||||
--sort_buffer_size=10M
|
||||
--max_connections=500
|
||||
ports:
|
||||
- "${MYSQL_PORT:-3306}:3306"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
- ./config/mysql-init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
|
||||
# Shared OpenSearch
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: distributed_test_opensearch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- plugins.security.disabled=true
|
||||
- "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS:--Xms512m -Xmx512m}"
|
||||
ports:
|
||||
- "${OPENSEARCH_PORT:-9200}:9200"
|
||||
- "9600:9600"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -qE '\"status\":\"(green|yellow)\"'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
|
||||
# Database migration (runs once)
|
||||
migrate:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_migrate
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
LOG_LEVEL: INFO
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- distributed-test-net
|
||||
|
||||
# OpenMetadata Server 1 (Primary - can trigger reindex)
|
||||
openmetadata-server-1:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_1
|
||||
hostname: om-server-1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-1
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8585/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
# OpenMetadata Server 2
|
||||
openmetadata-server-2:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_2
|
||||
hostname: om-server-2
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-2
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8587/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8587:8585"
|
||||
- "8588:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
# OpenMetadata Server 3
|
||||
openmetadata-server-3:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_3
|
||||
hostname: om-server-3
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-3
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8589/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8589:8585"
|
||||
- "8590:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Dependencies-only compose file for local JVM debugging
|
||||
# Use this when running OM servers from your IDE
|
||||
|
||||
version: "3.9"
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
distributed-test-net:
|
||||
name: distributed_test_network
|
||||
driver: bridge
|
||||
|
||||
services:
|
||||
# MySQL Database
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: distributed_test_mysql
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password}
|
||||
command: >
|
||||
--sort_buffer_size=10M
|
||||
--max_connections=500
|
||||
ports:
|
||||
- "${MYSQL_PORT:-3306}:3306"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
- ../config/mysql-init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
|
||||
# OpenSearch
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: distributed_test_opensearch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- plugins.security.disabled=true
|
||||
- "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS:--Xms512m -Xmx512m}"
|
||||
ports:
|
||||
- "${OPENSEARCH_PORT:-9200}:9200"
|
||||
- "9600:9600"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -qE '\"status\":\"(green|yellow)\"'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
# Run multiple OpenMetadata servers locally for debugging
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
# Default: run all 3 servers
|
||||
SERVERS="${@:-1 2 3}"
|
||||
|
||||
# Check if we should run migrations first
|
||||
RUN_MIGRATE=false
|
||||
if [ "$1" == "--migrate" ]; then
|
||||
RUN_MIGRATE=true
|
||||
shift
|
||||
SERVERS="${@:-1 2 3}"
|
||||
fi
|
||||
|
||||
echo "======================================"
|
||||
echo "Local Multi-Server Development Setup"
|
||||
echo "======================================"
|
||||
echo "Project dir: $PROJECT_DIR"
|
||||
echo "Servers to start: $SERVERS"
|
||||
echo ""
|
||||
|
||||
# Check if JAR exists
|
||||
JAR_PATH=$(find "$PROJECT_DIR/openmetadata-service/target" -name "openmetadata-service-*.jar" -not -name "*sources*" -not -name "*javadoc*" 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$JAR_PATH" ]; then
|
||||
echo "OpenMetadata JAR not found. Building..."
|
||||
cd "$PROJECT_DIR"
|
||||
mvn clean package -DskipTests -pl openmetadata-service -am
|
||||
JAR_PATH=$(find "$PROJECT_DIR/openmetadata-service/target" -name "openmetadata-service-*.jar" -not -name "*sources*" -not -name "*javadoc*" 2>/dev/null | head -1)
|
||||
fi
|
||||
|
||||
if [ -z "$JAR_PATH" ]; then
|
||||
echo "ERROR: Could not find or build openmetadata-service JAR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using JAR: $JAR_PATH"
|
||||
echo ""
|
||||
|
||||
# Check if dependencies are running
|
||||
echo "Checking dependencies..."
|
||||
if ! curl -s http://localhost:3306 >/dev/null 2>&1; then
|
||||
if ! docker ps | grep -q distributed_test_mysql; then
|
||||
echo "MySQL not running. Starting dependencies..."
|
||||
cd "$SCRIPT_DIR"
|
||||
docker compose -f docker-compose-deps.yml up -d
|
||||
echo "Waiting for MySQL..."
|
||||
until docker compose -f docker-compose-deps.yml exec -T mysql mysqladmin ping -h localhost -uroot -ppassword --silent 2>/dev/null; do
|
||||
sleep 2
|
||||
done
|
||||
echo "MySQL ready."
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! curl -s http://localhost:9200 >/dev/null 2>&1; then
|
||||
echo "OpenSearch not running. Please start dependencies first:"
|
||||
echo " cd $SCRIPT_DIR && docker compose -f docker-compose-deps.yml up -d"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Dependencies are running."
|
||||
echo ""
|
||||
|
||||
# Run migrations if requested
|
||||
if [ "$RUN_MIGRATE" == "true" ]; then
|
||||
echo "Running database migrations..."
|
||||
cd "$PROJECT_DIR"
|
||||
java -cp "$JAR_PATH" \
|
||||
-Dloader.main=org.openmetadata.service.util.OpenMetadataSetup \
|
||||
org.springframework.boot.loader.launch.PropertiesLauncher \
|
||||
migrate --force
|
||||
echo "Migrations complete."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Function to start a server
|
||||
start_server() {
|
||||
local server_num=$1
|
||||
local config_file="$SCRIPT_DIR/server${server_num}.yaml"
|
||||
|
||||
if [ ! -f "$config_file" ]; then
|
||||
echo "Config file not found: $config_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local port=$((8583 + server_num * 2))
|
||||
echo "Starting Server $server_num on port $port..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Start in a new terminal window (macOS)
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
osascript -e "tell application \"Terminal\" to do script \"cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -Ddw.logging.appenders[0].logFormat='[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n' -jar '$JAR_PATH' server '$config_file'\""
|
||||
# Linux with gnome-terminal
|
||||
elif command -v gnome-terminal &> /dev/null; then
|
||||
gnome-terminal -- bash -c "cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -Ddw.logging.appenders[0].logFormat='[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n' -jar '$JAR_PATH' server '$config_file'; exec bash"
|
||||
# Linux with xterm
|
||||
elif command -v xterm &> /dev/null; then
|
||||
xterm -e "cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -jar '$JAR_PATH' server '$config_file'" &
|
||||
else
|
||||
# Fallback: run in background
|
||||
echo "No terminal emulator found. Running in background..."
|
||||
java -Xmx1G -Xms512M \
|
||||
-Ddw.logging.appenders[0].logFormat="[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n" \
|
||||
-jar "$JAR_PATH" server "$config_file" \
|
||||
> "/tmp/om-server-$server_num.log" 2>&1 &
|
||||
echo "Server $server_num started in background. Log: /tmp/om-server-$server_num.log"
|
||||
fi
|
||||
}
|
||||
|
||||
# Start requested servers
|
||||
for server in $SERVERS; do
|
||||
start_server "$server"
|
||||
sleep 2 # Stagger startup
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "======================================"
|
||||
echo "Servers starting..."
|
||||
echo "======================================"
|
||||
echo ""
|
||||
echo "Server endpoints:"
|
||||
for server in $SERVERS; do
|
||||
port=$((8583 + server * 2))
|
||||
echo " - Server $server: http://localhost:$port"
|
||||
done
|
||||
echo ""
|
||||
echo "For IDE debugging:"
|
||||
echo " 1. Open your IDE"
|
||||
echo " 2. Create a new Run Configuration"
|
||||
echo " 3. Main class: org.openmetadata.service.OpenMetadataApplication"
|
||||
echo " 4. Program arguments: server local/server1.yaml"
|
||||
echo " 5. VM options: -Xmx1G -Xms512M"
|
||||
echo " 6. Working directory: $PROJECT_DIR"
|
||||
echo ""
|
||||
@@ -0,0 +1,639 @@
|
||||
# OpenMetadata Server 1 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server1.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8585}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8586}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-1
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-1] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-1
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: false
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: false
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,639 @@
|
||||
# OpenMetadata Server 2 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server2.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8587}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8588}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-2
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-2] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-2
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: ${NATURAL_LANGUAGE_SEARCH_ENABLED:-false}
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: ${BEDROCK_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,640 @@
|
||||
# OpenMetadata Server 3 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server3.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8589}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8590}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-3
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-3] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-3
|
||||
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: ${NATURAL_LANGUAGE_SEARCH_ENABLED:-false}
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: ${BEDROCK_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,96 @@
|
||||
version: "3.9"
|
||||
|
||||
# Compose override for RDF-enabled local stacks.
|
||||
# Use together with docker-compose.yml or docker-compose-postgres.yml.
|
||||
services:
|
||||
execute-migrate-all:
|
||||
environment:
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_DATASET: ${RDF_DATASET:-openmetadata}
|
||||
depends_on:
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
|
||||
openmetadata-server:
|
||||
environment:
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_DATASET: ${RDF_DATASET:-openmetadata}
|
||||
depends_on:
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
|
||||
fuseki:
|
||||
# Build from the in-repo Dockerfile (Fuseki 5.6.0) instead of the
|
||||
# unmaintained `stain/jena-fuseki` Docker Hub image, which capped at 5.1.0
|
||||
# and never picked up the 2025 admin-side Fuseki CVE fixes (CVE-2025-49656,
|
||||
# CVE-2025-50151 — both fixed in Jena 5.5.0). The `image:` tag below names
|
||||
# the locally-built image so subsequent `docker compose up` runs reuse the
|
||||
# cached build instead of rebuilding from scratch each time.
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: openmetadata-fuseki
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
networks:
|
||||
- local_app_net
|
||||
environment:
|
||||
# Default for local dev only — production deployments MUST override
|
||||
# via the FUSEKI_ADMIN_PASSWORD env var (and FUSEKI_OPENMETADATA_PASSWORD)
|
||||
# before bringing this stack up. The entrypoint envsubsts these into
|
||||
# shiro.ini at container start so the override actually takes effect.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=${FUSEKI_JVM_ARGS:--Xmx1500m -Xms256m}
|
||||
volumes:
|
||||
# New volume name (was `fuseki-data` mounted at `/fuseki`). The in-repo
|
||||
# Dockerfile stores TDB2 at `/fuseki-data` and the data layout differs
|
||||
# from the old stain/jena-fuseki image — re-using the previous volume
|
||||
# name would mount stale state at a path Fuseki no longer reads from,
|
||||
# silently looking like an empty database. Using a fresh volume name
|
||||
# forces operators to consciously migrate (or accept a re-index). The
|
||||
# orphaned `fuseki-data` volume can be removed manually with
|
||||
# `docker volume rm fuseki-data` after confirming the new stack is
|
||||
# healthy.
|
||||
- fuseki-tdb2-data:/fuseki-data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
reservations:
|
||||
memory: 256m
|
||||
restart: "on-failure:3"
|
||||
healthcheck:
|
||||
test: "curl -s -f http://localhost:3030/\\$/ping > /dev/null || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 60s
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
|
||||
volumes:
|
||||
fuseki-tdb2-data:
|
||||
driver: local
|
||||
@@ -0,0 +1,571 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
###
|
||||
# How to use this compose yaml file.
|
||||
# You have to prepare the credential json file for the service account which can access the GCP secret manager and set the path of the json file into the GOOGLE_APPLICATION_CREDENTIALS variable.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
mysql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/mysql/Dockerfile_mysql
|
||||
command: "--sort_buffer_size=10M"
|
||||
container_name: openmetadata_mysql
|
||||
restart: always
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
container_name: openmetadata_elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
# HACK: This is hack for M1 mac or later to avoid aborting JVM by the Google Secret Manager library.
|
||||
platform: linux/amd64
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: /key.json
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
# GCP:
|
||||
OM_SM_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
volumes:
|
||||
- ${GOOGLE_APPLICATION_CREDENTIALS}:/key.json:ro
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: /key.json
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
|
||||
## Secrets Manager
|
||||
# To integrate Azure Key Vault
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_KEY_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
# To Integrate with AWS SSM
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_REGION: ${OM_SM_AWS_REGION:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY_ID: ${OM_SM_AWS_ACCESS_KEY_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_SECRET_ACCESS_KEY: ${OM_SM_AWS_SECRET_ACCESS_KEY:-""}
|
||||
# To integrate GCP
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ${GOOGLE_APPLICATION_CREDENTIALS}:/key.json:ro
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,614 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
# See docker-compose-fuseki.yml — renamed from `fuseki-data` to avoid
|
||||
# silently mounting stale state under the new Fuseki layout.
|
||||
fuseki-tdb2-data:
|
||||
services:
|
||||
postgresql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
container_name: openmetadata_postgresql
|
||||
restart: always
|
||||
command: "--work_mem=10MB"
|
||||
depends_on:
|
||||
- opensearch
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: password
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: openmetadata_opensearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- plugins.security.disabled=true
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!!
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/opensearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for Postgres
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
fuseki:
|
||||
condition: service_started
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_started
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
fuseki:
|
||||
# See docker-compose-fuseki.yml for the rationale behind building from the
|
||||
# in-repo Dockerfile instead of using `stain/jena-fuseki:*` (unmaintained,
|
||||
# capped at 5.1.0, missing 2025 admin-side CVE fixes).
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: openmetadata-fuseki
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Local-dev default — production deployments MUST override via
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
volumes:
|
||||
# See docker-compose-fuseki.yml for why the volume was renamed from
|
||||
# `fuseki-data` to `fuseki-tdb2-data` (the data layout differs from the
|
||||
# previous stain/jena-fuseki image and reusing the old name silently
|
||||
# mounts stale state at a path Fuseki no longer reads).
|
||||
- fuseki-tdb2-data:/fuseki-data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,569 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
postgresql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
container_name: openmetadata_postgresql
|
||||
restart: always
|
||||
command: "--work_mem=10MB"
|
||||
depends_on:
|
||||
- opensearch
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: password
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: openmetadata_opensearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- plugins.security.disabled=true
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!!
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/opensearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
# Application connector type. Set SERVER_PROTOCOL=h2c on the host to enable
|
||||
# cleartext HTTP/2 at Jetty. Admin connector stays HTTP/1.1 regardless.
|
||||
SERVER_PROTOCOL: ${SERVER_PROTOCOL:-http}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for Postgres
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_started
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,8 @@
|
||||
# Override that disables the cache while leaving the rest of the stack intact.
|
||||
# Used in the local A/B benchmark to flip cache off without tearing down volumes.
|
||||
# Apply on TOP of base compose (NOT the redis overlay):
|
||||
# docker compose -f docker-compose.yml -f docker-compose.cache-off.yml up -d --no-deps openmetadata-server
|
||||
services:
|
||||
openmetadata-server:
|
||||
environment:
|
||||
CACHE_PROVIDER: none
|
||||
@@ -0,0 +1,86 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
# Adds a second OM instance that shares MySQL/Elasticsearch/Redis with the
|
||||
# primary one in docker-compose.yml. Used to validate that pub/sub
|
||||
# invalidation keeps per-instance Guava caches coherent.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.redis.yml \
|
||||
# -f docker-compose.multiserver.yml up -d
|
||||
services:
|
||||
openmetadata-server-2:
|
||||
image: development-openmetadata-server
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server_2
|
||||
restart: always
|
||||
networks:
|
||||
- local_app_net
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8587:8585"
|
||||
- "8588:8586"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: openmetadata
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: INFO
|
||||
FERNET_KEY: jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_USE_SSL: "false"
|
||||
DB_USER: openmetadata_user
|
||||
DB_USER_PASSWORD: openmetadata_password
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
OM_DATABASE: openmetadata_db
|
||||
ELASTICSEARCH_HOST: elasticsearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
SEARCH_TYPE: elasticsearch
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: openmetadata
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: "true"
|
||||
AUTHORIZER_CLASS_NAME: org.openmetadata.service.security.DefaultAuthorizer
|
||||
AUTHORIZER_REQUEST_FILTER: org.openmetadata.service.security.JwtFilter
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: open-metadata.org
|
||||
AUTHORIZER_ALLOWED_DOMAINS: "[]"
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: '["all"]'
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: "[ingestion-bot]"
|
||||
AUTHENTICATION_RESPONSE_TYPE: id_token
|
||||
AUTHENTICATION_CLIENT_TYPE: public
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://openmetadata-server-2:8585/api/v1/system/config/jwks]"
|
||||
AUTHENTICATION_AUTHORITY: https://accounts.google.com
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: "[email,preferred_username,sub]"
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ./conf/public_key.der
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ./conf/private_key.der
|
||||
JWT_ISSUER: open-metadata.org
|
||||
JWT_KEY_ID: Gb389a-9f76-gdjs-a92j-0242bk94356
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: http://ingestion:8080
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient
|
||||
AIRFLOW_USERNAME: admin
|
||||
AIRFLOW_PASSWORD: admin
|
||||
AIRFLOW_TIMEOUT: 10
|
||||
SECRET_MANAGER: db
|
||||
SERVER_HOST_API_URL: http://openmetadata-server-2:8585/api
|
||||
EVENT_MONITOR: prometheus
|
||||
OPENMETADATA_HEAP_OPTS: "-Xmx1G -Xms1G"
|
||||
CACHE_PROVIDER: redis
|
||||
CACHE_REDIS_URL: redis://redis:6379
|
||||
CACHE_REDIS_AUTH_TYPE: NONE
|
||||
CACHE_REDIS_KEYSPACE: om:dev
|
||||
CACHE_ENTITY_TTL: 3600
|
||||
CACHE_RELATIONSHIP_TTL: 3600
|
||||
CACHE_TAG_TTL: 3600
|
||||
CACHE_REDIS_COMMAND_TIMEOUT: 300
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
# Override that adds a Redis cache to the development stack.
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.redis.yml up -d
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: openmetadata_redis
|
||||
restart: always
|
||||
command: ["redis-server", "--appendonly", "no", "--save", "", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
networks:
|
||||
- local_app_net
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
openmetadata-server:
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
CACHE_PROVIDER: redis
|
||||
CACHE_REDIS_URL: redis://redis:6379
|
||||
CACHE_REDIS_AUTH_TYPE: NONE
|
||||
CACHE_REDIS_KEYSPACE: om:dev
|
||||
CACHE_ENTITY_TTL: 3600
|
||||
CACHE_RELATIONSHIP_TTL: 3600
|
||||
CACHE_TAG_TTL: 3600
|
||||
CACHE_REDIS_COMMAND_TIMEOUT: 300
|
||||
@@ -0,0 +1,652 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
fuseki-data:
|
||||
services:
|
||||
mysql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/mysql/Dockerfile_mysql
|
||||
command: "--sort_buffer_size=10M"
|
||||
container_name: openmetadata_mysql
|
||||
restart: always
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
container_name: openmetadata_elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=${ES_JAVA_OPTS:--Xms1024m -Xmx1024m}
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# RDF / SPARQL backend (optional; default off). Start fuseki with `--profile rdf`.
|
||||
RDF_ENABLED: ${RDF_ENABLED:-false}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-"http://fuseki:3030/openmetadata"}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
RDF_DATASET: ${RDF_DATASET:-"openmetadata"}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
# Application connector type. Set SERVER_PROTOCOL=h2c on the host to enable
|
||||
# cleartext HTTP/2 at Jetty. Admin connector stays HTTP/1.1 regardless.
|
||||
SERVER_PROTOCOL: ${SERVER_PROTOCOL:-http}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# RDF / SPARQL backend (optional; default off). Start fuseki with `--profile rdf`.
|
||||
RDF_ENABLED: ${RDF_ENABLED:-false}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-"http://fuseki:3030/openmetadata"}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
RDF_DATASET: ${RDF_DATASET:-"openmetadata"}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
# GCP:
|
||||
OM_SM_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__LOGGING__LOGGING_LEVEL: ${AIRFLOW_LOGGING_LEVEL:-DEBUG}
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
# OpenLineage transport config (optional - enable for lineage via OL)
|
||||
# AIRFLOW__OPENLINEAGE__TRANSPORT: '{"type": "http", "url": "http://openmetadata-server:8585/api/v1/openlineage/", "endpoint": "lineage", "auth": {"type": "api_key", "api_key": "<OM_JWT_TOKEN>"}}'
|
||||
# AIRFLOW__OPENLINEAGE__NAMESPACE: local_airflow
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
## Secrets Manager
|
||||
# To integrate Azure Key Vault
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_KEY_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
# To Integrate with AWS SSM
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_REGION: ${OM_SM_AWS_REGION:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY_ID: ${OM_SM_AWS_ACCESS_KEY_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_SECRET_ACCESS_KEY: ${OM_SM_AWS_SECRET_ACCESS_KEY:-""}
|
||||
# To integrate GCP
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
# Apps
|
||||
ENABLE_APP_HelloPipelines: "true"
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
mock-oidc-provider:
|
||||
build:
|
||||
context: ./mock-oidc-provider
|
||||
dockerfile: Dockerfile
|
||||
container_name: mock_oidc_provider
|
||||
environment:
|
||||
PORT: "9090"
|
||||
ISSUER: ${MOCK_OIDC_ISSUER:-http://localhost:9090}
|
||||
INTERNAL_BASE_URL: ${MOCK_OIDC_INTERNAL_BASE_URL:-http://mock-oidc-provider:9090}
|
||||
expose:
|
||||
- 9090
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
profiles:
|
||||
- sso-test
|
||||
|
||||
# Apache Jena Fuseki triplestore for the optional RDF/SPARQL backend (ontology
|
||||
# round-trip, semantic search). Start with `--profile rdf` and set RDF_ENABLED=true.
|
||||
# The glossary RDF import works without this; it only adds the transparent triplestore.
|
||||
fuseki:
|
||||
build:
|
||||
context: ../rdf-store
|
||||
container_name: openmetadata-fuseki
|
||||
environment:
|
||||
# The admin user carries the writer role; the server connects as
|
||||
# RDF_REMOTE_USERNAME=admin, so keep this password in sync with
|
||||
# RDF_REMOTE_PASSWORD. Dataset (/openmetadata) is baked into the image.
|
||||
- FUSEKI_ADMIN_PASSWORD=${RDF_REMOTE_PASSWORD:-admin}
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 3030
|
||||
ports:
|
||||
- "3030:3030"
|
||||
volumes:
|
||||
- fuseki-data:/fuseki-data
|
||||
profiles:
|
||||
- rdf
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,295 @@
|
||||
# OpenMetadata Helm Chart Local Testing
|
||||
|
||||
Helper to test changes from https://github.com/open-metadata/openmetadata-helm-charts with local images while developing.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Required Tools
|
||||
```bash
|
||||
# Install required tools
|
||||
brew install helm kubectl minikube docker-compose
|
||||
|
||||
# Or on Ubuntu/Debian:
|
||||
# sudo snap install helm kubectl minikube
|
||||
# sudo apt-get install docker-compose
|
||||
```
|
||||
|
||||
### Required Resources
|
||||
- **CPU**: 4+ cores recommended
|
||||
- **Memory**: 8GB+ RAM recommended
|
||||
- **Storage**: 20GB+ free space
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Start Local Dependencies
|
||||
|
||||
First, start the required database and search services:
|
||||
|
||||
```bash
|
||||
cd docker/development/helm
|
||||
|
||||
# Start PostgreSQL and OpenSearch
|
||||
docker-compose -f docker-compose-deps.yml up -d
|
||||
|
||||
# Wait for services to be ready (2-3 minutes)
|
||||
docker-compose -f docker-compose-deps.yml logs -f
|
||||
|
||||
# Verify services are running
|
||||
curl http://localhost:9200/_cluster/health
|
||||
docker exec openmetadata_postgres_test psql -U openmetadata_user -d openmetadata_db -c "SELECT 1"
|
||||
```
|
||||
|
||||
### 2. Start Kubernetes Cluster
|
||||
|
||||
```bash
|
||||
# Start minikube with sufficient resources
|
||||
minikube start --cpus 4 --memory 8192 --driver docker
|
||||
|
||||
# Enable ingress addon (optional)
|
||||
minikube addons enable ingress
|
||||
|
||||
# Verify cluster is ready
|
||||
kubectl cluster-info
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
### 3. Create Required Secrets
|
||||
|
||||
```bash
|
||||
# Create secrets that the chart expects
|
||||
kubectl create secret generic postgres-secrets \
|
||||
--from-literal=openmetadata-postgres-password=openmetadata_password
|
||||
|
||||
kubectl create secret generic airflow-secrets \
|
||||
--from-literal=openmetadata-airflow-password=admin
|
||||
```
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Scenario A: Test Kubernetes Native Pipeline Client (NEW)
|
||||
|
||||
Test the new K8s native pipeline execution without Airflow dependencies.
|
||||
|
||||
```bash
|
||||
# Use local chart directly, e.g.,
|
||||
CHART_PATH="/Users/pmbrull/github/openmetadata-helm-charts/charts/openmetadata"
|
||||
|
||||
# Install with K8s native configuration
|
||||
helm install openmetadata-k8s-test $CHART_PATH --values values-k8s-test.yaml
|
||||
|
||||
# Check deployment status
|
||||
kubectl get pods -A
|
||||
kubectl get jobs -n openmetadata-pipelines-test
|
||||
kubectl get serviceaccounts,roles,rolebindings -n openmetadata-pipelines-test
|
||||
|
||||
# Check logs
|
||||
kubectl logs -l app.kubernetes.io/name=openmetadata -f
|
||||
|
||||
# Test access
|
||||
minikube tunnel # In separate terminal
|
||||
curl http://localhost:8585/api/v1/system/health
|
||||
```
|
||||
|
||||
### Scenario B: Test Migrated Airflow Configuration
|
||||
|
||||
Test that the new nested Airflow configuration still works.
|
||||
|
||||
```bash
|
||||
# Install with migrated Airflow configuration
|
||||
helm install openmetadata-airflow-test $CHART_PATH \
|
||||
--values values-airflow-test.yaml \
|
||||
--timeout 10m \
|
||||
--wait
|
||||
|
||||
# Check deployment
|
||||
kubectl get pods -A
|
||||
kubectl logs -l app.kubernetes.io/name=openmetadata -f
|
||||
```
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
### ✅ Basic Functionality
|
||||
|
||||
1. **Pod Health**
|
||||
```bash
|
||||
# Check all pods are running
|
||||
kubectl get pods -A
|
||||
|
||||
# Check OpenMetadata pod logs
|
||||
kubectl logs deployment/openmetadata-k8s-test -f
|
||||
|
||||
# Check database connectivity
|
||||
kubectl exec deployment/openmetadata-k8s-test -- curl -f http://postgres:5432 || echo "DB connection test"
|
||||
```
|
||||
|
||||
2. **API Health**
|
||||
```bash
|
||||
# Access health endpoint
|
||||
curl http://localhost:8585/api/v1/system/health
|
||||
|
||||
# Check API response
|
||||
curl http://localhost:8585/api/v1/system/config
|
||||
```
|
||||
|
||||
### ✅ K8s Native Pipeline Features
|
||||
|
||||
1. **RBAC Resources**
|
||||
```bash
|
||||
# Check namespace creation
|
||||
kubectl get namespace openmetadata-pipelines-test
|
||||
|
||||
# Check service account
|
||||
kubectl get serviceaccount -n openmetadata-pipelines-test openmetadata-ingestion-test
|
||||
|
||||
# Check permissions
|
||||
kubectl auth can-i create jobs \
|
||||
--as=system:serviceaccount:openmetadata-pipelines-test:openmetadata-ingestion-test \
|
||||
-n openmetadata-pipelines-test
|
||||
```
|
||||
|
||||
2. **Configuration Validation**
|
||||
```bash
|
||||
# Check environment variables in pod
|
||||
kubectl exec deployment/openmetadata-k8s-test -- env | grep K8S_
|
||||
|
||||
# Check secrets
|
||||
kubectl get secret openmetadata-k8s-test-pipeline-secret -o yaml
|
||||
kubectl get secret openmetadata-k8s-test-pipeline-secret -o jsonpath='{.data}' | base64 -d
|
||||
```
|
||||
|
||||
3. **Pipeline Job Testing**
|
||||
```bash
|
||||
# Create a test pipeline job (manual)
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: test-ingestion-job
|
||||
namespace: openmetadata-pipelines-test
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: openmetadata-ingestion-test
|
||||
containers:
|
||||
- name: test
|
||||
image: docker.getcollate.io/openmetadata/ingestion:latest
|
||||
command: ["echo", "Test pipeline job works"]
|
||||
restartPolicy: Never
|
||||
EOF
|
||||
|
||||
# Check job execution
|
||||
kubectl get jobs -n openmetadata-pipelines-test
|
||||
kubectl logs job/test-ingestion-job -n openmetadata-pipelines-test
|
||||
```
|
||||
|
||||
### ✅ Failure Diagnostics Testing
|
||||
|
||||
1. **Create Failing Job**
|
||||
```bash
|
||||
# Create a job that will fail
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: test-failing-job
|
||||
namespace: openmetadata-pipelines-test
|
||||
labels:
|
||||
app.kubernetes.io/pipeline: test-pipeline
|
||||
app.kubernetes.io/run-id: test-123
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: openmetadata-ingestion-test
|
||||
containers:
|
||||
- name: main
|
||||
image: docker.getcollate.io/openmetadata/ingestion:latest
|
||||
command: ["sh", "-c", "echo 'Starting ingestion...'; sleep 10; echo 'Something went wrong!'; exit 1"]
|
||||
restartPolicy: Never
|
||||
EOF
|
||||
|
||||
# Watch for diagnostic job creation
|
||||
kubectl get jobs -n openmetadata-pipelines-test -w
|
||||
|
||||
# Check diagnostic job logs
|
||||
kubectl logs -n openmetadata-pipelines-test -l app.kubernetes.io/component=diagnostics
|
||||
```
|
||||
|
||||
### ✅ Configuration Migration Testing
|
||||
|
||||
1. **Test Both Configurations**
|
||||
```bash
|
||||
# Test that both airflow and k8s configs are valid
|
||||
helm template test-airflow $CHART_PATH --values values-airflow-test.yaml > /tmp/airflow-manifest.yaml
|
||||
helm template test-k8s $CHART_PATH --values values-k8s-test.yaml > /tmp/k8s-manifest.yaml
|
||||
|
||||
# Validate manifests
|
||||
kubectl apply --dry-run=client -f /tmp/airflow-manifest.yaml
|
||||
kubectl apply --dry-run=client -f /tmp/k8s-manifest.yaml
|
||||
```
|
||||
|
||||
2. **Test Breaking Changes**
|
||||
```bash
|
||||
# Test old configuration format (should fail validation or show warnings)
|
||||
cat > /tmp/old-values.yaml << EOF
|
||||
openmetadata:
|
||||
config:
|
||||
pipelineServiceClientConfig:
|
||||
enabled: true
|
||||
className: "org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"
|
||||
apiEndpoint: http://test
|
||||
EOF
|
||||
|
||||
helm template test-old $CHART_PATH --values /tmp/old-values.yaml
|
||||
```
|
||||
|
||||
### Debug Commands
|
||||
|
||||
```bash
|
||||
# Get all resources
|
||||
kubectl get all -A
|
||||
|
||||
# Describe OpenMetadata deployment
|
||||
kubectl describe deployment openmetadata-k8s-test
|
||||
|
||||
# Get events
|
||||
kubectl get events --sort-by='.lastTimestamp' -A
|
||||
|
||||
# Check helm release
|
||||
helm list
|
||||
helm status openmetadata-k8s-test
|
||||
helm get values openmetadata-k8s-test
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
### Remove Test Deployments
|
||||
```bash
|
||||
# Remove helm releases
|
||||
helm uninstall openmetadata-k8s-test
|
||||
helm uninstall openmetadata-airflow-test
|
||||
|
||||
# Clean up namespaces
|
||||
kubectl delete namespace openmetadata-pipelines-test
|
||||
|
||||
# Clean up secrets
|
||||
kubectl delete secret postgres-secrets airflow-secrets
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
### Successful Deployment Indicators
|
||||
|
||||
1. ✅ **All pods running**: `kubectl get pods -A` shows all pods in `Running` state
|
||||
2. ✅ **Health check passing**: `curl http://localhost:8585/api/v1/system/health` returns 200
|
||||
3. ✅ **RBAC created**: K8s resources created in `openmetadata-pipelines-test` namespace
|
||||
4. ✅ **Configuration loaded**: Environment variables properly set in pods
|
||||
5. ✅ **No errors in logs**: `kubectl logs deployment/openmetadata-k8s-test` shows successful startup
|
||||
|
||||
### Performance Benchmarks
|
||||
|
||||
- **Startup time**: < 5 minutes for full deployment
|
||||
- **Memory usage**: < 4GB total (including dependencies)
|
||||
- **CPU usage**: < 2 cores under normal load
|
||||
- **Job creation**: < 30 seconds for pipeline job creation and execution
|
||||
|
||||
This comprehensive testing suite validates both the new K8s native functionality and ensures backward compatibility with existing Airflow configurations.
|
||||
@@ -0,0 +1,82 @@
|
||||
# Docker compose for OpenMetadata dependencies (PostgreSQL + OpenSearch)
|
||||
# Use this for local Helm chart testing
|
||||
|
||||
version: "3.9"
|
||||
services:
|
||||
|
||||
# PostgreSQL Database
|
||||
postgres:
|
||||
container_name: openmetadata_postgres_test
|
||||
build:
|
||||
context: ../../../
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_DB: openmetadata_db
|
||||
POSTGRES_USER: openmetadata_user
|
||||
POSTGRES_PASSWORD: openmetadata_password
|
||||
POSTGRES_ROOT_PASSWORD: password
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5433:5432" # Use different port to avoid conflicts
|
||||
networks:
|
||||
- openmetadata_network
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
command: >
|
||||
postgres
|
||||
-c shared_preload_libraries=pg_stat_statements
|
||||
-c max_connections=200
|
||||
-c shared_buffers=256MB
|
||||
-c effective_cache_size=1GB
|
||||
-c maintenance_work_mem=64MB
|
||||
-c checkpoint_completion_target=0.9
|
||||
-c wal_buffers=16MB
|
||||
-c default_statistics_target=100
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openmetadata_user -d openmetadata_db"]
|
||||
timeout: 20s
|
||||
retries: 10
|
||||
interval: 30s
|
||||
|
||||
# OpenSearch for metadata search
|
||||
opensearch:
|
||||
container_name: openmetadata_opensearch_test
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
restart: always
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- node.name=opensearch
|
||||
- cluster.name=opensearch-cluster
|
||||
- bootstrap.memory_lock=true
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m"
|
||||
- "DISABLE_INSTALL_DEMO_CONFIG=true"
|
||||
- "DISABLE_SECURITY_PLUGIN=true"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
networks:
|
||||
- openmetadata_network
|
||||
volumes:
|
||||
- opensearch_data:/usr/share/opensearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
opensearch_data:
|
||||
|
||||
networks:
|
||||
openmetadata_network:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,67 @@
|
||||
# Test values for migrated Airflow pipeline client validation
|
||||
# Only contains essential overrides for testing the new nested structure
|
||||
|
||||
openmetadata:
|
||||
config:
|
||||
# Enable debug logging for testing
|
||||
logLevel: DEBUG
|
||||
|
||||
# Database configuration (using local PostgreSQL via host machine)
|
||||
database:
|
||||
host: host.docker.internal
|
||||
port: 5433
|
||||
driverClass: org.postgresql.Driver
|
||||
dbScheme: postgresql
|
||||
auth:
|
||||
password:
|
||||
secretRef: postgres-secrets
|
||||
secretKey: openmetadata-postgres-password
|
||||
|
||||
# Search configuration (using local OpenSearch via host machine)
|
||||
elasticsearch:
|
||||
host: host.docker.internal
|
||||
port: 9200
|
||||
searchType: opensearch
|
||||
scheme: http
|
||||
|
||||
# Airflow Pipeline Client Configuration (updated structure with common configs)
|
||||
pipelineServiceClientConfig:
|
||||
type: "airflow" # Use traditional Airflow approach
|
||||
# Common configuration for all pipeline service clients
|
||||
metadataApiEndpoint: http://openmetadata:8585/api
|
||||
airflow:
|
||||
# Airflow-specific connection settings
|
||||
apiEndpoint: http://openmetadata-dependencies-api-server:8080
|
||||
verifySsl: "no-ssl"
|
||||
auth:
|
||||
username: admin
|
||||
password:
|
||||
secretRef: airflow-secrets
|
||||
secretKey: openmetadata-airflow-password
|
||||
|
||||
# Local image configuration
|
||||
image:
|
||||
repository: openmetadata/server
|
||||
tag: "20260107"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Service configuration for local access
|
||||
service:
|
||||
type: LoadBalancer # For minikube tunnel access
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# More relaxed health checks for local testing
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
startupProbe:
|
||||
failureThreshold: 10
|
||||
@@ -0,0 +1,112 @@
|
||||
# Test values for K8s native pipeline client validation
|
||||
# Only contains essential overrides for testing
|
||||
|
||||
openmetadata:
|
||||
config:
|
||||
# Enable debug logging for testing
|
||||
logLevel: DEBUG
|
||||
|
||||
# Database configuration (using local PostgreSQL via host machine)
|
||||
database:
|
||||
host: host.docker.internal
|
||||
port: 5432
|
||||
driverClass: org.postgresql.Driver
|
||||
dbScheme: postgresql
|
||||
auth:
|
||||
password:
|
||||
secretRef: postgres-secrets
|
||||
secretKey: openmetadata-postgres-password
|
||||
|
||||
# Search configuration (using local OpenSearch via host machine)
|
||||
elasticsearch:
|
||||
host: host.docker.internal
|
||||
port: 9200
|
||||
searchType: opensearch
|
||||
scheme: http
|
||||
|
||||
# K8s Pipeline Client Configuration - Updated structure with common configs
|
||||
pipelineServiceClientConfig:
|
||||
type: "k8s" # Use Kubernetes native instead of Airflow
|
||||
# Common configuration for all pipeline service clients
|
||||
# Override metadataApiEndpoint for cross-namespace access (default uses service name only)
|
||||
metadataApiEndpoint: "http://openmetadata.openmetadata-ingestion.svc.cluster.local:8585/api"
|
||||
k8s:
|
||||
# K8s-specific configuration
|
||||
# Use release namespace (empty = defaults to release namespace)
|
||||
namespace: ""
|
||||
serviceAccountName: "openmetadata-ingestion"
|
||||
ttlSecondsAfterFinished: 10
|
||||
|
||||
# Local ingestion image
|
||||
ingestionImage: "openmetadata/collate-base:1.12.0-20260402"
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "2Gi"
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "512Mi"
|
||||
|
||||
# Test tolerations for ingestion pods
|
||||
tolerations:
|
||||
- key: "dedicated"
|
||||
operator: "Equal"
|
||||
value: "ingestion"
|
||||
effect: "NoSchedule"
|
||||
|
||||
# Enable OMJob operator for guaranteed exit handler execution
|
||||
useOMJobOperator: true
|
||||
|
||||
# Local image configuration
|
||||
image:
|
||||
repository: openmetadata/server
|
||||
tag: "1.12.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Service configuration for local access
|
||||
service:
|
||||
type: LoadBalancer # For minikube tunnel access
|
||||
|
||||
# OMJob Operator configuration
|
||||
omjobOperator:
|
||||
enabled: true
|
||||
# Use locally built fixed image
|
||||
image:
|
||||
repository: docker.getcollate.io/openmetadata/omjob-operator
|
||||
tag: "1.12.0-SNAPSHOT"
|
||||
pullPolicy: IfNotPresent
|
||||
# Adjusted resources for Java application (was OOMKilled at 128Mi)
|
||||
resources:
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "1Gi"
|
||||
# Debug logging for testing
|
||||
env:
|
||||
logLevel: "DEBUG"
|
||||
watchNamespaces: ""
|
||||
# Enable health checks now that operator implements proper endpoints
|
||||
healthCheck:
|
||||
enabled: true
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# More relaxed health checks for local testing
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
startupProbe:
|
||||
failureThreshold: 10
|
||||
@@ -0,0 +1,12 @@
|
||||
FROM node:20-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json ./
|
||||
RUN npm install --production
|
||||
|
||||
COPY server.js ./
|
||||
|
||||
EXPOSE 9090
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "mock-oidc-provider",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "Mock OIDC identity provider for OpenMetadata E2E SSO testing",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"oidc-provider": "^8.4.6",
|
||||
"jose": "^5.2.4",
|
||||
"express": "^4.18.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* Copyright 2025 Collate.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const { Provider } = require('oidc-provider');
|
||||
const express = require('express');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '9090', 10);
|
||||
const ISSUER = process.env.ISSUER || `http://localhost:${PORT}`;
|
||||
|
||||
// Mutable test state — controlled via /test/* endpoints
|
||||
const DEFAULT_LOGIN_ACCOUNT = 'admin';
|
||||
|
||||
const testState = {
|
||||
accessTokenTTL: 3600,
|
||||
idTokenTTL: 3600,
|
||||
forceInteractionRequired: false,
|
||||
refreshTokenEnabled: true,
|
||||
tokenEndpointError: null, // { errorCode: 'invalid_grant', httpStatus: 400 }
|
||||
// Account auto-approved on the next interactive login. Lets a test exercise a
|
||||
// specific identity (e.g. one whose sub differs from the email local-part).
|
||||
defaultLoginAccount: DEFAULT_LOGIN_ACCOUNT,
|
||||
};
|
||||
|
||||
const resetTestState = () => {
|
||||
testState.accessTokenTTL = 3600;
|
||||
testState.idTokenTTL = 3600;
|
||||
testState.forceInteractionRequired = false;
|
||||
testState.refreshTokenEnabled = true;
|
||||
testState.tokenEndpointError = null;
|
||||
testState.defaultLoginAccount = DEFAULT_LOGIN_ACCOUNT;
|
||||
};
|
||||
|
||||
// Request metrics — reset via /test/metrics/reset
|
||||
const metrics = { tokenRequests: 0, authRequests: 0, refreshAttempts: 0 };
|
||||
|
||||
// Pre-seeded test accounts — auto-approved, no interactive login
|
||||
const TEST_ACCOUNTS = new Map([
|
||||
[
|
||||
'admin',
|
||||
{
|
||||
email: 'admin@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Test Admin',
|
||||
sub: 'admin',
|
||||
preferred_username: 'admin',
|
||||
},
|
||||
],
|
||||
[
|
||||
'user1',
|
||||
{
|
||||
email: 'user1@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Test User 1',
|
||||
sub: 'user1',
|
||||
preferred_username: 'user1',
|
||||
},
|
||||
],
|
||||
// Identity whose sub ('claim-user') deliberately differs from the email
|
||||
// local-part ('claim.user.mapped'). Used to verify OIDC self-signup persists
|
||||
// the mapped email claim instead of deriving <sub>@<domain>. See issue #29189.
|
||||
[
|
||||
'claim-user',
|
||||
{
|
||||
email: 'claim.user.mapped@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Claim Mapped User',
|
||||
sub: 'claim-user',
|
||||
preferred_username: 'claim-user',
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const findAccount = (_ctx, id) => {
|
||||
const account = TEST_ACCOUNTS.get(id);
|
||||
if (!account) return undefined;
|
||||
return {
|
||||
accountId: id,
|
||||
async claims(_use, _scope) {
|
||||
return { sub: id, ...account };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const clients = [
|
||||
{
|
||||
client_id: 'openmetadata-test',
|
||||
client_secret: 'openmetadata-test-secret',
|
||||
grant_types: ['authorization_code', 'refresh_token'],
|
||||
redirect_uris: [
|
||||
'http://localhost:8585/callback',
|
||||
'http://localhost:3000/callback',
|
||||
'http://localhost:8585/silent-callback',
|
||||
],
|
||||
post_logout_redirect_uris: [
|
||||
'http://localhost:8585',
|
||||
'http://localhost:3000',
|
||||
],
|
||||
response_types: ['code'],
|
||||
token_endpoint_auth_method: 'client_secret_post',
|
||||
scope: 'openid email profile offline_access',
|
||||
},
|
||||
{
|
||||
client_id: 'openmetadata-test-public',
|
||||
grant_types: ['authorization_code', 'refresh_token'],
|
||||
redirect_uris: [
|
||||
'http://localhost:8585/callback',
|
||||
'http://localhost:3000/callback',
|
||||
'http://localhost:8585/silent-callback',
|
||||
],
|
||||
post_logout_redirect_uris: [
|
||||
'http://localhost:8585',
|
||||
'http://localhost:3000',
|
||||
],
|
||||
response_types: ['code'],
|
||||
token_endpoint_auth_method: 'none',
|
||||
scope: 'openid email profile offline_access',
|
||||
},
|
||||
];
|
||||
|
||||
const providerConfig = {
|
||||
clients,
|
||||
findAccount,
|
||||
claims: {
|
||||
openid: ['sub'],
|
||||
email: ['email', 'email_verified'],
|
||||
profile: ['name', 'preferred_username'],
|
||||
},
|
||||
scopes: ['openid', 'email', 'profile', 'offline_access'],
|
||||
features: {
|
||||
devInteractions: { enabled: false },
|
||||
rpInitiatedLogout: { enabled: true },
|
||||
},
|
||||
pkce: {
|
||||
required: () => false,
|
||||
},
|
||||
ttl: {
|
||||
AccessToken: (_ctx, _token, _client) => testState.accessTokenTTL,
|
||||
IdToken: (_ctx, _token, _client) => testState.idTokenTTL,
|
||||
RefreshToken: (_ctx, _token, _client) =>
|
||||
testState.refreshTokenEnabled ? 86400 : 0,
|
||||
AuthorizationCode: 600,
|
||||
Session: 86400,
|
||||
Grant: 86400,
|
||||
Interaction: 600,
|
||||
},
|
||||
cookies: {
|
||||
keys: [crypto.randomBytes(32).toString('hex')],
|
||||
},
|
||||
jwks: {
|
||||
keys: [
|
||||
{
|
||||
kty: 'RSA',
|
||||
kid: 'mock-oidc-key-1',
|
||||
use: 'sig',
|
||||
alg: 'RS256',
|
||||
// Generated at startup; see init() below
|
||||
},
|
||||
],
|
||||
},
|
||||
// Auto-approve all interactions (skip login/consent screens)
|
||||
interactions: {
|
||||
url(_ctx, _interaction) {
|
||||
return `/interaction/${_interaction.uid}`;
|
||||
},
|
||||
},
|
||||
renderError: async (ctx, out, _error) => {
|
||||
ctx.type = 'html';
|
||||
ctx.body = `<html><body><pre>${JSON.stringify(out, null, 2)}</pre></body></html>`;
|
||||
},
|
||||
};
|
||||
|
||||
async function init() {
|
||||
const { generateKeyPair, exportJWK } = await import('jose');
|
||||
const { privateKey, publicKey } = await generateKeyPair('RS256');
|
||||
const privateJwk = await exportJWK(privateKey);
|
||||
const publicJwk = await exportJWK(publicKey);
|
||||
|
||||
privateJwk.kid = 'mock-oidc-key-1';
|
||||
privateJwk.use = 'sig';
|
||||
privateJwk.alg = 'RS256';
|
||||
|
||||
publicJwk.kid = 'mock-oidc-key-1';
|
||||
publicJwk.use = 'sig';
|
||||
publicJwk.alg = 'RS256';
|
||||
|
||||
providerConfig.jwks = { keys: [privateJwk] };
|
||||
|
||||
const provider = new Provider(ISSUER, providerConfig);
|
||||
|
||||
// Auto-approve interaction: when the OIDC library redirects to /interaction/:uid,
|
||||
// we automatically log in as 'admin' and grant consent without any user input.
|
||||
provider.use(async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
if (
|
||||
ctx.path.startsWith('/interaction/') &&
|
||||
ctx.method === 'GET' &&
|
||||
!ctx.path.endsWith('/abort')
|
||||
) {
|
||||
const uid = ctx.path.split('/interaction/')[1];
|
||||
if (!uid || uid.includes('/')) return;
|
||||
|
||||
try {
|
||||
const interactionDetails = await provider.interactionDetails(
|
||||
ctx.req,
|
||||
ctx.res
|
||||
);
|
||||
const { prompt } = interactionDetails;
|
||||
|
||||
if (testState.forceInteractionRequired) {
|
||||
testState.forceInteractionRequired = false;
|
||||
ctx.status = 403;
|
||||
ctx.body = JSON.stringify({
|
||||
error: 'interaction_required',
|
||||
error_description:
|
||||
'Forced interaction_required for testing silent renewal failure',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (prompt.name === 'login') {
|
||||
const loginUser =
|
||||
ctx.query.login_hint ||
|
||||
interactionDetails.params.login_hint ||
|
||||
testState.defaultLoginAccount ||
|
||||
'admin';
|
||||
const result = {
|
||||
login: { accountId: loginUser },
|
||||
};
|
||||
await provider.interactionFinished(ctx.req, ctx.res, result, {
|
||||
mergeWithLastSubmission: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (prompt.name === 'consent') {
|
||||
const grant = new provider.Grant({
|
||||
accountId: interactionDetails.session.accountId,
|
||||
clientId: interactionDetails.params.client_id,
|
||||
});
|
||||
|
||||
const requestedScopes = interactionDetails.params.scope;
|
||||
if (requestedScopes) {
|
||||
grant.addOIDCScope(requestedScopes);
|
||||
}
|
||||
grant.addOIDCScope('openid email profile offline_access');
|
||||
|
||||
const grantId = await grant.save();
|
||||
|
||||
const result = { consent: { grantId } };
|
||||
await provider.interactionFinished(ctx.req, ctx.res, result, {
|
||||
mergeWithLastSubmission: true,
|
||||
});
|
||||
}
|
||||
} catch (_err) {
|
||||
// Interaction may have already been handled
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
|
||||
// Server-facing base URL for endpoints that the OM server calls from
|
||||
// inside Docker (token exchange, JWKS, userinfo). Defaults to ISSUER
|
||||
// so that outside-Docker usage (tests hitting localhost) works unchanged.
|
||||
const INTERNAL_BASE =
|
||||
process.env.INTERNAL_BASE_URL || ISSUER;
|
||||
|
||||
// Custom discovery endpoint returning hybrid URLs:
|
||||
// - Browser-facing (authorization, end_session): ISSUER (localhost:9090)
|
||||
// - Server-facing (token, jwks, userinfo): INTERNAL_BASE (mock-oidc-provider:9090 in Docker)
|
||||
// This is mounted before oidc-provider so it takes precedence.
|
||||
app.get('/.well-known/openid-configuration', (_req, res) => {
|
||||
res.json({
|
||||
issuer: ISSUER,
|
||||
authorization_endpoint: `${ISSUER}/auth`,
|
||||
token_endpoint: `${INTERNAL_BASE}/token`,
|
||||
jwks_uri: `${INTERNAL_BASE}/jwks`,
|
||||
userinfo_endpoint: `${INTERNAL_BASE}/me`,
|
||||
end_session_endpoint: `${ISSUER}/session/end`,
|
||||
pushed_authorization_request_endpoint: `${INTERNAL_BASE}/request`,
|
||||
claims_parameter_supported: false,
|
||||
claims_supported: [
|
||||
'sub',
|
||||
'email',
|
||||
'email_verified',
|
||||
'name',
|
||||
'preferred_username',
|
||||
'sid',
|
||||
'auth_time',
|
||||
'iss',
|
||||
],
|
||||
code_challenge_methods_supported: ['S256'],
|
||||
grant_types_supported: [
|
||||
'implicit',
|
||||
'authorization_code',
|
||||
'refresh_token',
|
||||
],
|
||||
response_modes_supported: ['form_post', 'fragment', 'query'],
|
||||
response_types_supported: ['code id_token', 'code', 'id_token', 'none'],
|
||||
scopes_supported: ['openid', 'email', 'profile', 'offline_access'],
|
||||
subject_types_supported: ['public'],
|
||||
id_token_signing_alg_values_supported: ['RS256'],
|
||||
token_endpoint_auth_methods_supported: [
|
||||
'client_secret_basic',
|
||||
'client_secret_jwt',
|
||||
'client_secret_post',
|
||||
'private_key_jwt',
|
||||
'none',
|
||||
],
|
||||
token_endpoint_auth_signing_alg_values_supported: [
|
||||
'HS256',
|
||||
'RS256',
|
||||
'PS256',
|
||||
'ES256',
|
||||
'EdDSA',
|
||||
],
|
||||
authorization_response_iss_parameter_supported: true,
|
||||
request_parameter_supported: false,
|
||||
request_uri_parameter_supported: false,
|
||||
claim_types_supported: ['normal'],
|
||||
});
|
||||
});
|
||||
|
||||
// Test control endpoints
|
||||
app.post('/test/configure', (req, res) => {
|
||||
const {
|
||||
accessTokenTTL,
|
||||
idTokenTTL,
|
||||
refreshTokenEnabled,
|
||||
forceInteractionRequired,
|
||||
tokenEndpointError,
|
||||
defaultLoginAccount,
|
||||
} = req.body;
|
||||
if (accessTokenTTL !== undefined)
|
||||
testState.accessTokenTTL = accessTokenTTL;
|
||||
if (idTokenTTL !== undefined) testState.idTokenTTL = idTokenTTL;
|
||||
if (refreshTokenEnabled !== undefined)
|
||||
testState.refreshTokenEnabled = refreshTokenEnabled;
|
||||
if (forceInteractionRequired !== undefined)
|
||||
testState.forceInteractionRequired = forceInteractionRequired;
|
||||
if (tokenEndpointError !== undefined)
|
||||
testState.tokenEndpointError = tokenEndpointError;
|
||||
if (defaultLoginAccount !== undefined)
|
||||
testState.defaultLoginAccount = defaultLoginAccount;
|
||||
|
||||
res.json({ ok: true, state: { ...testState } });
|
||||
});
|
||||
|
||||
app.post('/test/force-interaction-required', (_req, res) => {
|
||||
testState.forceInteractionRequired = true;
|
||||
res.json({ ok: true, forceInteractionRequired: true });
|
||||
});
|
||||
|
||||
app.post('/test/reset', (_req, res) => {
|
||||
resetTestState();
|
||||
res.json({ ok: true, state: { ...testState } });
|
||||
});
|
||||
|
||||
app.get('/test/state', (_req, res) => {
|
||||
res.json({ ...testState });
|
||||
});
|
||||
|
||||
// Metrics endpoints
|
||||
app.get('/test/metrics', (_req, res) => {
|
||||
res.json({ ...metrics });
|
||||
});
|
||||
|
||||
app.post('/test/metrics/reset', (_req, res) => {
|
||||
metrics.tokenRequests = 0;
|
||||
metrics.authRequests = 0;
|
||||
metrics.refreshAttempts = 0;
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// Token endpoint interceptor — must be BEFORE oidc-provider mount.
|
||||
// Tracks metrics and optionally injects errors for testing.
|
||||
app.post('/token', (req, res, next) => {
|
||||
metrics.tokenRequests++;
|
||||
|
||||
const grantType = req.body?.grant_type;
|
||||
if (grantType === 'refresh_token') {
|
||||
metrics.refreshAttempts++;
|
||||
}
|
||||
|
||||
if (testState.tokenEndpointError) {
|
||||
const { errorCode, httpStatus } = testState.tokenEndpointError;
|
||||
testState.tokenEndpointError = null; // one-shot
|
||||
return res.status(httpStatus).json({ error: errorCode });
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
// Auth endpoint interceptor — tracks authorization request metrics.
|
||||
app.get('/auth', (req, res, next) => {
|
||||
metrics.authRequests++;
|
||||
next();
|
||||
});
|
||||
|
||||
// Health check
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({ status: 'ok' });
|
||||
});
|
||||
|
||||
// Mount the OIDC provider
|
||||
app.use(provider.callback());
|
||||
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(
|
||||
`Mock OIDC Provider listening on ${ISSUER}`
|
||||
);
|
||||
console.log(
|
||||
`Discovery: ${ISSUER}/.well-known/openid-configuration`
|
||||
);
|
||||
console.log(`Test control: POST ${ISSUER}/test/configure`);
|
||||
});
|
||||
}
|
||||
|
||||
init().catch((err) => {
|
||||
console.error('Failed to start mock OIDC provider:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user