chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,50 @@
|
||||
# SSO Test Environment Configuration
|
||||
# Configures OM server to use mock OIDC provider for E2E testing.
|
||||
#
|
||||
# Usage:
|
||||
# 1. Start mock OIDC provider first:
|
||||
# docker compose --profile sso-test up -d mock-oidc-provider
|
||||
#
|
||||
# 2. Start (or restart) OM server with this env file:
|
||||
# docker compose --env-file .env.sso-test up -d
|
||||
#
|
||||
# 3. Run SSO Playwright tests:
|
||||
# cd openmetadata-ui/src/main/resources/ui
|
||||
# npx playwright test --config=playwright.sso.config.ts
|
||||
#
|
||||
# NOTE: Server-side URLs use Docker hostname (mock-oidc-provider:9090).
|
||||
# Client-side URLs (authority, callback) use localhost:9090.
|
||||
|
||||
# Authentication provider — custom-oidc uses OidcAuthenticator
|
||||
AUTHENTICATION_PROVIDER=custom-oidc
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=mock-oidc
|
||||
AUTHENTICATION_CLIENT_TYPE=confidential
|
||||
AUTHENTICATION_RESPONSE_TYPE=code
|
||||
|
||||
# Public keys — fetched server-side, must use Docker network hostname
|
||||
AUTHENTICATION_PUBLIC_KEYS=[http://mock-oidc-provider:9090/jwks,http://localhost:8585/api/v1/system/config/jwks]
|
||||
# Authority — sent to browser for redirect, must use localhost
|
||||
AUTHENTICATION_AUTHORITY=http://localhost:9090
|
||||
AUTHENTICATION_CLIENT_ID=openmetadata-test
|
||||
AUTHENTICATION_CALLBACK_URL=http://localhost:8585/callback
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP=true
|
||||
# Map principal claims: username from `sub`, email from the `email` claim.
|
||||
# Exercises the OIDC self-signup mapped-email path (issue #29189).
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING=[username:sub,email:email]
|
||||
|
||||
# OIDC Configuration — discovery URI is fetched server-side
|
||||
OIDC_CLIENT_ID=openmetadata-test
|
||||
OIDC_TYPE=custom-oidc
|
||||
OIDC_CLIENT_SECRET=openmetadata-test-secret
|
||||
OIDC_SCOPE=openid email profile
|
||||
OIDC_DISCOVERY_URI=http://mock-oidc-provider:9090/.well-known/openid-configuration
|
||||
OIDC_USE_NONCE=true
|
||||
OIDC_PREFERRED_JWS=RS256
|
||||
OIDC_RESPONSE_TYPE=code
|
||||
OIDC_DISABLE_PKCE=true
|
||||
OIDC_CALLBACK=http://localhost:8585/callback
|
||||
OIDC_SERVER_URL=http://localhost:8585
|
||||
OIDC_CLIENT_AUTH_METHOD=client_secret_post
|
||||
OIDC_SESSION_EXPIRY=604800
|
||||
AUTHENTICATION_SESSION_EXPIRY=604800
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER=5
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build stage
|
||||
FROM alpine:3 AS build
|
||||
|
||||
COPY openmetadata-dist/target/openmetadata-*.tar.gz /
|
||||
|
||||
RUN mkdir -p /opt/openmetadata && \
|
||||
tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
|
||||
rm openmetadata-*.tar.gz
|
||||
|
||||
# Final stage
|
||||
FROM alpine:3
|
||||
|
||||
EXPOSE 8585
|
||||
|
||||
RUN adduser -D openmetadata && \
|
||||
apk update && \
|
||||
apk upgrade && \
|
||||
apk add --update --no-cache bash openjdk21-jre
|
||||
|
||||
COPY --chown=openmetadata:openmetadata --from=build /opt/openmetadata /opt/openmetadata
|
||||
COPY --chmod=755 docker/openmetadata-start.sh /
|
||||
|
||||
USER openmetadata
|
||||
|
||||
WORKDIR /opt/openmetadata
|
||||
ENTRYPOINT [ "/bin/bash" ]
|
||||
CMD ["/openmetadata-start.sh"]
|
||||
@@ -0,0 +1,253 @@
|
||||
# Distributed Search Indexing Test Environment
|
||||
|
||||
This directory contains scripts and configurations to test the distributed search indexing feature with multiple OpenMetadata servers sharing a common database.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Test Environment │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ OM Server│ │ OM Server│ │ OM Server│ │
|
||||
│ │ :8585 │ │ :8587 │ │ :8589 │ │
|
||||
│ │ SERVER-1 │ │ SERVER-2 │ │ SERVER-3 │ │
|
||||
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
|
||||
│ │ │ │ │
|
||||
│ └─────────────┼─────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────┴──────┐ │
|
||||
│ │ Polling │ (DB-based coordination) │
|
||||
│ └──────┬──────┘ │
|
||||
│ │ │
|
||||
│ ┌─────────────┴─────────────┐ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌─────────┐ ┌───────────┐ │
|
||||
│ │ MySQL │ │ OpenSearch│ │
|
||||
│ │ :3306 │ │ :9200 │ │
|
||||
│ └─────────┘ └───────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Quick Start (Docker Compose)
|
||||
|
||||
### 1. Start the Environment
|
||||
|
||||
```bash
|
||||
cd docker/development/distributed-test
|
||||
|
||||
# Start all services (builds images on first run)
|
||||
./scripts/start.sh
|
||||
|
||||
# Or force rebuild
|
||||
./scripts/start.sh --build
|
||||
```
|
||||
|
||||
### 2. Load Test Data
|
||||
|
||||
```bash
|
||||
# Load 10,000 tables (default)
|
||||
./scripts/perf-test.sh
|
||||
|
||||
# Or specify the number
|
||||
./scripts/perf-test.sh --tables 50000 --databases 50
|
||||
```
|
||||
|
||||
### 3. Trigger Reindexing
|
||||
|
||||
```bash
|
||||
# Trigger reindex on server 1
|
||||
./scripts/trigger-reindex.sh
|
||||
|
||||
# With index recreation
|
||||
./scripts/trigger-reindex.sh --recreate
|
||||
|
||||
# Specific entities only
|
||||
./scripts/trigger-reindex.sh --entities table,dashboard
|
||||
```
|
||||
|
||||
### 4. Monitor Progress
|
||||
|
||||
```bash
|
||||
# Follow logs from all servers
|
||||
./scripts/logs.sh -f
|
||||
|
||||
# Filter by pattern
|
||||
./scripts/logs.sh -f --grep "partition"
|
||||
|
||||
# Single server logs
|
||||
./scripts/logs.sh -f --server 1
|
||||
```
|
||||
|
||||
### 5. Stop the Environment
|
||||
|
||||
```bash
|
||||
# Stop containers (preserve data)
|
||||
./scripts/stop.sh
|
||||
|
||||
# Stop and clean up volumes
|
||||
./scripts/stop.sh --clean
|
||||
```
|
||||
|
||||
## Local Development (IDE Debugging)
|
||||
|
||||
For debugging with breakpoints, run OM servers locally while using Docker for MySQL and OpenSearch.
|
||||
|
||||
### 1. Start Dependencies Only
|
||||
|
||||
```bash
|
||||
cd docker/development/distributed-test
|
||||
docker compose -f local/docker-compose-deps.yml up -d
|
||||
```
|
||||
|
||||
### 2. Run Migrations (First Time)
|
||||
|
||||
```bash
|
||||
cd /path/to/openmetadata
|
||||
./bootstrap/openmetadata-ops.sh -d migrate --force
|
||||
```
|
||||
|
||||
### 3. Option A: Run from Terminal
|
||||
|
||||
```bash
|
||||
# Start all 3 servers in separate terminals
|
||||
./local/run-local-servers.sh
|
||||
|
||||
# Or specific servers
|
||||
./local/run-local-servers.sh 1 2
|
||||
```
|
||||
|
||||
### 3. Option B: Run from IDE
|
||||
|
||||
Create run configurations in IntelliJ IDEA:
|
||||
|
||||
**Server 1:**
|
||||
- Main class: `org.openmetadata.service.OpenMetadataApplication`
|
||||
- Program arguments: `server docker/development/distributed-test/local/server1.yaml`
|
||||
- VM options: `-Xmx1G -Xms512M`
|
||||
- Working directory: Project root
|
||||
|
||||
**Server 2:**
|
||||
- Same as above but with `local/server2.yaml`
|
||||
|
||||
**Server 3:**
|
||||
- Same as above but with `local/server3.yaml`
|
||||
|
||||
## Server Ports
|
||||
|
||||
| Server | API Port | Admin Port |
|
||||
|--------|----------|------------|
|
||||
| Server 1 | 8585 | 8586 |
|
||||
| Server 2 | 8587 | 8588 |
|
||||
| Server 3 | 8589 | 8590 |
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `.env` to customize:
|
||||
|
||||
```bash
|
||||
# Number of tables for test data
|
||||
TEST_DATA_TABLES=10000
|
||||
|
||||
# Log level
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Heap size per server
|
||||
OPENMETADATA_HEAP_OPTS=-Xmx1G -Xms1G
|
||||
```
|
||||
|
||||
## Testing Distributed Indexing
|
||||
|
||||
### Verify Partition Distribution
|
||||
|
||||
1. Start all 3 servers
|
||||
2. Load test data: `./scripts/perf-test.sh --tables 10000`
|
||||
3. Trigger reindex: `./scripts/trigger-reindex.sh --recreate`
|
||||
4. Watch logs: `./scripts/logs.sh -f --grep "partition"`
|
||||
|
||||
You should see output like:
|
||||
```
|
||||
[SERVER-1] INFO Claimed partition: table_0-999 (1000 records)
|
||||
[SERVER-2] INFO Claimed partition: table_1000-1999 (1000 records)
|
||||
[SERVER-3] INFO Claimed partition: table_2000-2999 (1000 records)
|
||||
[SERVER-1] INFO Completed partition: table_0-999
|
||||
...
|
||||
```
|
||||
|
||||
### Test Server Failure Recovery
|
||||
|
||||
1. Start reindexing with many partitions
|
||||
2. Stop one server mid-process: `docker stop distributed_test_om_server_2`
|
||||
3. Watch remaining servers pick up orphaned partitions
|
||||
4. Verify job completes successfully
|
||||
|
||||
### Check Job Status
|
||||
|
||||
```bash
|
||||
# Via API
|
||||
curl -s http://localhost:8585/api/v1/apps/name/SearchIndexingApplication/status | jq
|
||||
|
||||
# Check partition table directly
|
||||
docker exec -it distributed_test_mysql mysql -uopenmetadata_user -popenmetadata_password openmetadata_db \
|
||||
-e "SELECT status, COUNT(*) FROM search_index_partition GROUP BY status"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Servers Not Starting
|
||||
|
||||
Check if ports are in use:
|
||||
```bash
|
||||
lsof -i :8585
|
||||
lsof -i :8587
|
||||
lsof -i :8589
|
||||
```
|
||||
|
||||
### Database Connection Issues
|
||||
|
||||
Verify MySQL is accessible:
|
||||
```bash
|
||||
docker exec -it distributed_test_mysql mysql -uopenmetadata_user -popenmetadata_password -e "SELECT 1"
|
||||
```
|
||||
|
||||
### OpenSearch Not Ready
|
||||
|
||||
Check health:
|
||||
```bash
|
||||
curl http://localhost:9200/_cluster/health?pretty
|
||||
```
|
||||
|
||||
### View Full Logs
|
||||
|
||||
```bash
|
||||
# All container logs
|
||||
docker compose logs -f
|
||||
|
||||
# Specific container
|
||||
docker logs -f distributed_test_om_server_1
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
distributed-test/
|
||||
├── docker-compose.yml # Full environment (3 OM servers + deps)
|
||||
├── .env # Configuration variables
|
||||
├── config/
|
||||
│ └── mysql-init.sql # Database initialization
|
||||
├── scripts/
|
||||
│ ├── start.sh # Start full environment
|
||||
│ ├── stop.sh # Stop environment
|
||||
│ ├── logs.sh # View aggregated logs
|
||||
│ ├── trigger-reindex.sh # Trigger reindexing
|
||||
│ └── perf-test.sh # Load test data
|
||||
├── local/
|
||||
│ ├── docker-compose-deps.yml # Dependencies only (for IDE debugging)
|
||||
│ ├── server1.yaml # Server 1 config (port 8585)
|
||||
│ ├── server2.yaml # Server 2 config (port 8587)
|
||||
│ ├── server3.yaml # Server 3 config (port 8589)
|
||||
│ └── run-local-servers.sh # Start servers locally
|
||||
└── README.md # This file
|
||||
```
|
||||
@@ -0,0 +1,13 @@
|
||||
-- MySQL initialization script for distributed test environment
|
||||
|
||||
-- Create the OpenMetadata database
|
||||
CREATE DATABASE IF NOT EXISTS openmetadata_db;
|
||||
|
||||
-- Create the OpenMetadata user
|
||||
CREATE USER IF NOT EXISTS 'openmetadata_user'@'%' IDENTIFIED BY 'openmetadata_password';
|
||||
|
||||
-- Grant privileges
|
||||
GRANT ALL PRIVILEGES ON openmetadata_db.* TO 'openmetadata_user'@'%';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openmetadata_user'@'%';
|
||||
|
||||
FLUSH PRIVILEGES;
|
||||
@@ -0,0 +1,270 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Distributed Search Indexing Test Environment
|
||||
# This compose file sets up multiple OM servers sharing a common MySQL and OpenSearch
|
||||
|
||||
version: "3.9"
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
distributed-test-net:
|
||||
name: distributed_test_network
|
||||
driver: bridge
|
||||
|
||||
services:
|
||||
# Shared MySQL Database
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: distributed_test_mysql
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password}
|
||||
command: >
|
||||
--sort_buffer_size=10M
|
||||
--max_connections=500
|
||||
ports:
|
||||
- "${MYSQL_PORT:-3306}:3306"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
- ./config/mysql-init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
|
||||
# Shared OpenSearch
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: distributed_test_opensearch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- plugins.security.disabled=true
|
||||
- "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS:--Xms512m -Xmx512m}"
|
||||
ports:
|
||||
- "${OPENSEARCH_PORT:-9200}:9200"
|
||||
- "9600:9600"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -qE '\"status\":\"(green|yellow)\"'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
|
||||
# Database migration (runs once)
|
||||
migrate:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_migrate
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
LOG_LEVEL: INFO
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- distributed-test-net
|
||||
|
||||
# OpenMetadata Server 1 (Primary - can trigger reindex)
|
||||
openmetadata-server-1:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_1
|
||||
hostname: om-server-1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-1
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8585/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
# OpenMetadata Server 2
|
||||
openmetadata-server-2:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_2
|
||||
hostname: om-server-2
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-2
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8587/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8587:8585"
|
||||
- "8588:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
# OpenMetadata Server 3
|
||||
openmetadata-server-3:
|
||||
build:
|
||||
context: ../../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: distributed_test_om_server_3
|
||||
hostname: om-server-3
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
OPENMETADATA_CLUSTER_NAME: distributed-test
|
||||
|
||||
# Unique server identifier for distributed indexing
|
||||
OM_SERVER_ID: server-3
|
||||
|
||||
# Database
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
|
||||
# Search
|
||||
SEARCH_TYPE: opensearch
|
||||
ELASTICSEARCH_HOST: opensearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
|
||||
# Auth
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://localhost:8589/api/v1/system/config/jwks]"
|
||||
|
||||
# Pipeline service disabled for testing
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: "false"
|
||||
|
||||
# Heap
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
ports:
|
||||
- "8589:8585"
|
||||
- "8590:8586"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Dependencies-only compose file for local JVM debugging
|
||||
# Use this when running OM servers from your IDE
|
||||
|
||||
version: "3.9"
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
opensearch-data:
|
||||
|
||||
networks:
|
||||
distributed-test-net:
|
||||
name: distributed_test_network
|
||||
driver: bridge
|
||||
|
||||
services:
|
||||
# MySQL Database
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: distributed_test_mysql
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-password}
|
||||
command: >
|
||||
--sort_buffer_size=10M
|
||||
--max_connections=500
|
||||
ports:
|
||||
- "${MYSQL_PORT:-3306}:3306"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD:-password}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
- ../config/mysql-init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
|
||||
# OpenSearch
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: distributed_test_opensearch
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- plugins.security.disabled=true
|
||||
- "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS:--Xms512m -Xmx512m}"
|
||||
ports:
|
||||
- "${OPENSEARCH_PORT:-9200}:9200"
|
||||
- "9600:9600"
|
||||
networks:
|
||||
- distributed-test-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -qE '\"status\":\"(green|yellow)\"'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
volumes:
|
||||
- opensearch-data:/usr/share/opensearch/data
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
# Run multiple OpenMetadata servers locally for debugging
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
# Default: run all 3 servers
|
||||
SERVERS="${@:-1 2 3}"
|
||||
|
||||
# Check if we should run migrations first
|
||||
RUN_MIGRATE=false
|
||||
if [ "$1" == "--migrate" ]; then
|
||||
RUN_MIGRATE=true
|
||||
shift
|
||||
SERVERS="${@:-1 2 3}"
|
||||
fi
|
||||
|
||||
echo "======================================"
|
||||
echo "Local Multi-Server Development Setup"
|
||||
echo "======================================"
|
||||
echo "Project dir: $PROJECT_DIR"
|
||||
echo "Servers to start: $SERVERS"
|
||||
echo ""
|
||||
|
||||
# Check if JAR exists
|
||||
JAR_PATH=$(find "$PROJECT_DIR/openmetadata-service/target" -name "openmetadata-service-*.jar" -not -name "*sources*" -not -name "*javadoc*" 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$JAR_PATH" ]; then
|
||||
echo "OpenMetadata JAR not found. Building..."
|
||||
cd "$PROJECT_DIR"
|
||||
mvn clean package -DskipTests -pl openmetadata-service -am
|
||||
JAR_PATH=$(find "$PROJECT_DIR/openmetadata-service/target" -name "openmetadata-service-*.jar" -not -name "*sources*" -not -name "*javadoc*" 2>/dev/null | head -1)
|
||||
fi
|
||||
|
||||
if [ -z "$JAR_PATH" ]; then
|
||||
echo "ERROR: Could not find or build openmetadata-service JAR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using JAR: $JAR_PATH"
|
||||
echo ""
|
||||
|
||||
# Check if dependencies are running
|
||||
echo "Checking dependencies..."
|
||||
if ! curl -s http://localhost:3306 >/dev/null 2>&1; then
|
||||
if ! docker ps | grep -q distributed_test_mysql; then
|
||||
echo "MySQL not running. Starting dependencies..."
|
||||
cd "$SCRIPT_DIR"
|
||||
docker compose -f docker-compose-deps.yml up -d
|
||||
echo "Waiting for MySQL..."
|
||||
until docker compose -f docker-compose-deps.yml exec -T mysql mysqladmin ping -h localhost -uroot -ppassword --silent 2>/dev/null; do
|
||||
sleep 2
|
||||
done
|
||||
echo "MySQL ready."
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! curl -s http://localhost:9200 >/dev/null 2>&1; then
|
||||
echo "OpenSearch not running. Please start dependencies first:"
|
||||
echo " cd $SCRIPT_DIR && docker compose -f docker-compose-deps.yml up -d"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Dependencies are running."
|
||||
echo ""
|
||||
|
||||
# Run migrations if requested
|
||||
if [ "$RUN_MIGRATE" == "true" ]; then
|
||||
echo "Running database migrations..."
|
||||
cd "$PROJECT_DIR"
|
||||
java -cp "$JAR_PATH" \
|
||||
-Dloader.main=org.openmetadata.service.util.OpenMetadataSetup \
|
||||
org.springframework.boot.loader.launch.PropertiesLauncher \
|
||||
migrate --force
|
||||
echo "Migrations complete."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Function to start a server
|
||||
start_server() {
|
||||
local server_num=$1
|
||||
local config_file="$SCRIPT_DIR/server${server_num}.yaml"
|
||||
|
||||
if [ ! -f "$config_file" ]; then
|
||||
echo "Config file not found: $config_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local port=$((8583 + server_num * 2))
|
||||
echo "Starting Server $server_num on port $port..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Start in a new terminal window (macOS)
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
osascript -e "tell application \"Terminal\" to do script \"cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -Ddw.logging.appenders[0].logFormat='[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n' -jar '$JAR_PATH' server '$config_file'\""
|
||||
# Linux with gnome-terminal
|
||||
elif command -v gnome-terminal &> /dev/null; then
|
||||
gnome-terminal -- bash -c "cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -Ddw.logging.appenders[0].logFormat='[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n' -jar '$JAR_PATH' server '$config_file'; exec bash"
|
||||
# Linux with xterm
|
||||
elif command -v xterm &> /dev/null; then
|
||||
xterm -e "cd '$PROJECT_DIR' && java -Xmx1G -Xms512M -jar '$JAR_PATH' server '$config_file'" &
|
||||
else
|
||||
# Fallback: run in background
|
||||
echo "No terminal emulator found. Running in background..."
|
||||
java -Xmx1G -Xms512M \
|
||||
-Ddw.logging.appenders[0].logFormat="[SERVER-$server_num] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n" \
|
||||
-jar "$JAR_PATH" server "$config_file" \
|
||||
> "/tmp/om-server-$server_num.log" 2>&1 &
|
||||
echo "Server $server_num started in background. Log: /tmp/om-server-$server_num.log"
|
||||
fi
|
||||
}
|
||||
|
||||
# Start requested servers
|
||||
for server in $SERVERS; do
|
||||
start_server "$server"
|
||||
sleep 2 # Stagger startup
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "======================================"
|
||||
echo "Servers starting..."
|
||||
echo "======================================"
|
||||
echo ""
|
||||
echo "Server endpoints:"
|
||||
for server in $SERVERS; do
|
||||
port=$((8583 + server * 2))
|
||||
echo " - Server $server: http://localhost:$port"
|
||||
done
|
||||
echo ""
|
||||
echo "For IDE debugging:"
|
||||
echo " 1. Open your IDE"
|
||||
echo " 2. Create a new Run Configuration"
|
||||
echo " 3. Main class: org.openmetadata.service.OpenMetadataApplication"
|
||||
echo " 4. Program arguments: server local/server1.yaml"
|
||||
echo " 5. VM options: -Xmx1G -Xms512M"
|
||||
echo " 6. Working directory: $PROJECT_DIR"
|
||||
echo ""
|
||||
@@ -0,0 +1,639 @@
|
||||
# OpenMetadata Server 1 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server1.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8585}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8586}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-1
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-1] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-1
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: false
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: false
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,639 @@
|
||||
# OpenMetadata Server 2 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server2.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8587}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8588}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-2
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-2] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-2
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: ${NATURAL_LANGUAGE_SEARCH_ENABLED:-false}
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: ${BEDROCK_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,640 @@
|
||||
# OpenMetadata Server 3 Configuration for Local Development
|
||||
# Run with: java -Dconfig=local/server3.yaml -jar openmetadata-service/target/openmetadata-service-*.jar server
|
||||
|
||||
clusterName: distributed-test
|
||||
|
||||
swagger:
|
||||
resourcePackage: org.openmetadata.service.resources
|
||||
|
||||
assets:
|
||||
resourcePath: /assets/
|
||||
uriPath: /
|
||||
|
||||
basePath: ${BASE_PATH:-/}
|
||||
|
||||
server:
|
||||
applicationContextPath: /
|
||||
rootPath: /api/*
|
||||
applicationConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_PORT:-8589}
|
||||
|
||||
# Jetty 12 URI Compliance - UNSAFE allows all special characters in entity names
|
||||
# Required for backward compatibility with entity names containing /, ", etc.
|
||||
uriCompliance: UNSAFE
|
||||
|
||||
# Jetty Acceptor and Selector threads for high concurrency
|
||||
acceptorThreads: ${SERVER_ACCEPTOR_THREADS:-2} # 1-2 per CPU core
|
||||
selectorThreads: ${SERVER_SELECTOR_THREADS:-8} # 2-4 per CPU core
|
||||
|
||||
# Connection settings - relaxed for Docker/local development
|
||||
acceptQueueSize: ${SERVER_ACCEPT_QUEUE_SIZE:-256} # OS-level connection backlog
|
||||
idleTimeout: ${SERVER_IDLE_TIMEOUT:-60 seconds} # Close idle connections (increased from 30s)
|
||||
|
||||
# Buffer sizes for better throughput
|
||||
outputBufferSize: ${SERVER_OUTPUT_BUFFER_SIZE:-32KiB}
|
||||
inputBufferSize: ${SERVER_INPUT_BUFFER_SIZE:-8KiB}
|
||||
maxRequestHeaderSize: ${SERVER_MAX_REQUEST_HEADER_SIZE:-8KiB}
|
||||
maxResponseHeaderSize: ${SERVER_MAX_RESPONSE_HEADER_SIZE:-8KiB}
|
||||
headerCacheSize: ${SERVER_HEADER_CACHE_SIZE:-512B} # Cache parsed headers (in bytes)
|
||||
|
||||
# Performance settings
|
||||
useServerHeader: false # Don't send server version header
|
||||
useDateHeader: true
|
||||
useForwardedHeaders: ${SERVER_USE_FORWARDED_HEADERS:-false} # Enable if behind proxy
|
||||
|
||||
# Data rate limits (prevent slow loris attacks)
|
||||
minRequestDataPerSecond: ${SERVER_MIN_REQUEST_DATA_RATE:-0B} # 0B = disabled
|
||||
minResponseDataPerSecond: ${SERVER_MIN_RESPONSE_DATA_RATE:-0B} # 0B = disabled
|
||||
|
||||
adminConnectors:
|
||||
- type: http
|
||||
bindHost: ${SERVER_HOST:-0.0.0.0}
|
||||
port: ${SERVER_ADMIN_PORT:-8590}
|
||||
acceptorThreads: 1 # Admin endpoint needs minimal resources
|
||||
selectorThreads: 1
|
||||
|
||||
# Response compression disabled for maximum throughput
|
||||
gzip:
|
||||
enabled: false
|
||||
|
||||
# Thread pool configuration
|
||||
# With virtual threads enabled (Java 21+), these settings become less critical
|
||||
# as blocking operations are handled efficiently by the JVM
|
||||
maxThreads: ${SERVER_MAX_THREADS:-150}
|
||||
minThreads: ${SERVER_MIN_THREADS:-100}
|
||||
idleThreadTimeout: ${SERVER_IDLE_THREAD_TIMEOUT:-1 minute}
|
||||
|
||||
# Virtual Threads (Project Loom) - Recommended for Java 21+
|
||||
# Jetty 12 uses AdaptiveExecutionStrategy to route blocking tasks to virtual threads
|
||||
# while keeping non-blocking I/O on platform threads for optimal cache locality
|
||||
enableVirtualThreads: ${SERVER_ENABLE_VIRTUAL_THREAD:-false}
|
||||
# Note: maxQueuedRequests removed in Dropwizard 5.0/Jetty 12
|
||||
|
||||
# Request/Response logging (disable in production for performance)
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
requestLog:
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: ${REQUEST_LOG_LEVEL:-ERROR} # Only log errors by default
|
||||
layout:
|
||||
type: om-access-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-3
|
||||
|
||||
# Above configuration for running http is fine for dev and testing.
|
||||
# For production setup, where UI app will hit apis through DPS it
|
||||
# is strongly recommended to run https instead. Note that only
|
||||
# keyStorePath and keyStorePassword are mandatory properties. Values
|
||||
# for other properties are defaults
|
||||
#server:
|
||||
#applicationConnectors:
|
||||
# - type: https
|
||||
# port: 8585
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
#adminConnectors:
|
||||
# - type: https
|
||||
# port: 8586
|
||||
# keyStorePath: ./conf/keystore.jks
|
||||
# keyStorePassword: changeit
|
||||
# keyStoreType: JKS
|
||||
# keyStoreProvider:
|
||||
# trustStorePath: /path/to/file
|
||||
# trustStorePassword: changeit
|
||||
# trustStoreType: JKS
|
||||
# trustStoreProvider:
|
||||
# keyManagerPassword: changeit
|
||||
# needClientAuth: false
|
||||
# wantClientAuth:
|
||||
# certAlias: <alias>
|
||||
# crlPath: /path/to/file
|
||||
# enableCRLDP: false
|
||||
# enableOCSP: false
|
||||
# maxCertPathLength: (unlimited)
|
||||
# ocspResponderUrl: (none)
|
||||
# jceProvider: (none)
|
||||
# validateCerts: true
|
||||
# validatePeers: true
|
||||
# supportedProtocols: SSLv3
|
||||
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
|
||||
# allowRenegotiation: true
|
||||
# endpointIdentificationAlgorithm: (none)
|
||||
|
||||
# Logging settings.
|
||||
# https://logback.qos.ch/manual/layouts.html#conversionWord
|
||||
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.
|
||||
logging:
|
||||
level: ${LOG_LEVEL:-INFO}
|
||||
appenders:
|
||||
- type: console
|
||||
threshold: INFO
|
||||
layout:
|
||||
type: om-event-layout
|
||||
format: ${LOG_FORMAT:-text}
|
||||
pattern: "[server-3] %level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
|
||||
appendLineSeparator: true
|
||||
additionalFields:
|
||||
server: server-3
|
||||
|
||||
|
||||
database:
|
||||
# the name of the JDBC driver, mysql in our case
|
||||
driverClass: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
# the username and password
|
||||
user: ${DB_USER:-openmetadata_user}
|
||||
password: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
# the JDBC URL; the database is called openmetadata_db
|
||||
url: jdbc:${DB_SCHEME:-mysql}://${DB_HOST:-localhost}:${DB_PORT:-3306}/${OM_DATABASE:-openmetadata_db}?${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
|
||||
# HikariCP Connection Pool Settings - Optimized for Performance
|
||||
maxSize: ${DB_CONNECTION_POOL_MAX_SIZE:-100} # Increased from 50 for better concurrency
|
||||
minSize: ${DB_CONNECTION_POOL_MIN_SIZE:-20} # Increased from 10 to reduce connection creation overhead
|
||||
minimumIdle: ${DB_CONNECTION_POOL_MIN_IDLE:-20} # HikariCP specific minimum idle connections
|
||||
initialSize: ${DB_CONNECTION_POOL_INITIAL_SIZE:-20} # Start with more connections ready
|
||||
checkConnectionWhileIdle: ${DB_CONNECTION_CHECK_CONNECTION_WHILE_IDLE:-true}
|
||||
checkConnectionOnBorrow: ${DB_CONNECTION_CHECK_CONNECTION_ON_BORROW:-false} # Disable for performance
|
||||
evictionInterval: ${DB_CONNECTION_EVICTION_INTERVAL:-5 minutes}
|
||||
minIdleTime: ${DB_CONNECTION_MIN_IDLE_TIME:-5 minute}
|
||||
|
||||
|
||||
# JDBC Driver Properties - Critical for Performance
|
||||
# These work across both PostgreSQL and MySQL drivers
|
||||
properties:
|
||||
# HikariCP connection pool settings
|
||||
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
|
||||
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
|
||||
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
|
||||
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
|
||||
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
|
||||
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
|
||||
# PostgreSQL specific - these are ignored by MySQL driver
|
||||
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
|
||||
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
|
||||
preparedStatementCacheSizeMiB: ${DB_PG_PREP_STMT_CACHE_SIZE_MB:-10} # Larger cache
|
||||
reWriteBatchedInserts: ${DB_PG_REWRITE_BATCHED_INSERTS:-true} # Critical for batch performance
|
||||
defaultRowFetchSize: ${DB_PG_DEFAULT_ROW_FETCH_SIZE:-1000} # Fetch more rows at once
|
||||
assumeMinServerVersion: ${DB_PG_ASSUME_MIN_SERVER_VERSION:-12} # Skip version checks
|
||||
ApplicationName: ${DB_PG_APPLICATION_NAME:-OpenMetadata}
|
||||
loginTimeout: ${DB_PG_LOGIN_TIMEOUT:-300} # Login timeout in seconds
|
||||
postgresqlConnectTimeout: ${DB_POSTGRESQL_CONNECT_TIMEOUT:-60} # Connection timeout in seconds
|
||||
postgresqlSocketTimeout: ${DB_POSTGRESQL_SOCKET_TIMEOUT:-30000} # Socket timeout in seconds (0 = infinite)
|
||||
|
||||
# Aurora PostgreSQL specific optimizations
|
||||
loadBalanceHosts: ${DB_PG_LOAD_BALANCE_HOSTS:-false} # Set to true for Aurora reader endpoints
|
||||
hostRecheckSeconds: ${DB_PG_HOST_RECHECK_SECONDS:-10} # How often to check host status
|
||||
targetServerType: ${DB_PG_TARGET_SERVER_TYPE:-primary} # primary, secondary, any, preferSecondary
|
||||
|
||||
# MySQL specific - these are ignored by PostgreSQL driver
|
||||
rewriteBatchedStatements: ${DB_MYSQL_REWRITE_BATCHED_STATEMENTS:-true} # Critical for MySQL batch
|
||||
cachePrepStmts: ${DB_MYSQL_CACHE_PREP_STMTS:-true}
|
||||
prepStmtCacheSize: ${DB_MYSQL_PREP_STMT_CACHE_SIZE:-500}
|
||||
prepStmtCacheSqlLimit: ${DB_MYSQL_PREP_STMT_CACHE_SQL_LIMIT:-2048}
|
||||
useServerPrepStmts: ${DB_MYSQL_USE_SERVER_PREP_STMTS:-true}
|
||||
useLocalSessionState: ${DB_MYSQL_USE_LOCAL_SESSION_STATE:-true}
|
||||
useLocalTransactionState: ${DB_MYSQL_USE_LOCAL_TRANSACTION_STATE:-true}
|
||||
elideSetAutoCommits: ${DB_MYSQL_ELIDE_SET_AUTO_COMMITS:-true}
|
||||
maintainTimeStats: ${DB_MYSQL_MAINTAIN_TIME_STATS:-false}
|
||||
cacheResultSetMetadata: ${DB_MYSQL_CACHE_RESULT_SET_METADATA:-true}
|
||||
cacheServerConfiguration: ${DB_MYSQL_CACHE_SERVER_CONFIG:-true}
|
||||
tcpKeepAlive: ${DB_MYSQL_TCP_KEEP_ALIVE:-true}
|
||||
tcpNoDelay: ${DB_MYSQL_TCP_NO_DELAY:-true}
|
||||
mysqlConnectTimeout: ${DB_MYSQL_CONNECT_TIMEOUT:-60000} # Connection timeout in milliseconds
|
||||
mysqlSocketTimeout: ${DB_MYSQL_SOCKET_TIMEOUT:-30000000} # Socket timeout in milliseconds (0 = infinite)
|
||||
|
||||
objectStorage:
|
||||
enabled: false
|
||||
provider: NOOP
|
||||
maxFileSize: 5242880
|
||||
|
||||
migrationConfiguration:
|
||||
flywayPath: "./bootstrap/sql/migrations/flyway"
|
||||
nativePath: "./bootstrap/sql/migrations/native"
|
||||
extensionPath: ""
|
||||
|
||||
# Authorizer Configuration
|
||||
authorizerConfiguration:
|
||||
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
allowedDomains: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
enableSecureSocketConnection : ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
|
||||
|
||||
authenticationConfiguration:
|
||||
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
provider: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
# This is used by auth provider provide response as either id_token or code
|
||||
responseType: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
# This will only be valid when provider type specified is customOidc
|
||||
providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
|
||||
authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
clientId: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
|
||||
# Force secure flag on session cookies even when not using HTTPS directly.
|
||||
# Enable this when running behind a proxy/load balancer that handles SSL termination.
|
||||
# Default: false (secure flag only set when HTTPS is detected)
|
||||
forceSecureSessionCookie: ${FORCE_SECURE_SESSION_COOKIE:-false}
|
||||
sessionExpiry: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"} # 7 days; applies to all auth providers
|
||||
maxActiveSessionsPerUser: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
oidcConfiguration:
|
||||
id: ${OIDC_CLIENT_ID:-""}
|
||||
type: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
secret: ${OIDC_CLIENT_SECRET:-""}
|
||||
scope: ${OIDC_SCOPE:-"openid email profile"}
|
||||
discoveryUri: ${OIDC_DISCOVERY_URI:-""}
|
||||
useNonce: ${OIDC_USE_NONCE:-true}
|
||||
preferredJwsAlgorithm: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
responseType: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
disablePkce: ${OIDC_DISABLE_PKCE:-true}
|
||||
callbackUrl: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
serverUrl: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
clientAuthenticationMethod: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
tenant: ${OIDC_TENANT:-""}
|
||||
maxClockSkew: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
tokenValidity: ${OIDC_OM_REFRESH_TOKEN_VALIDITY:-"3600"} # in seconds
|
||||
customParams: ${OIDC_CUSTOM_PARAMS:-}
|
||||
maxAge: ${OIDC_MAX_AGE:-"0"}
|
||||
prompt: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
sessionExpiry: ${OIDC_SESSION_EXPIRY:-"604800"} #7 days
|
||||
samlConfiguration:
|
||||
debugMode: ${SAML_DEBUG_MODE:-false}
|
||||
idp:
|
||||
entityId: ${SAML_IDP_ENTITY_ID:-""}
|
||||
ssoLoginUrl: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
idpX509Certificate: ${SAML_IDP_CERTIFICATE:-""}
|
||||
nameId: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
sp:
|
||||
entityId: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
acs: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
spX509Certificate: ${SAML_SP_CERTIFICATE:-""}
|
||||
spPrivateKey: ${SAML_SP_PRIVATE_KEY:-""}
|
||||
callback: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
security:
|
||||
strictMode: ${SAML_STRICT_MODE:-false}
|
||||
validateXml: ${SAML_VALIDATE_XML:-false}
|
||||
tokenValidity: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
sendEncryptedNameId: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
sendSignedAuthRequest: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
signSpMetadata: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
wantMessagesSigned: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
wantAssertionsSigned: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
wantAssertionEncrypted: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
keyStoreFilePath: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
keyStoreAlias: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
keyStorePassword: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
ldapConfiguration:
|
||||
host: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
port: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
dnAdminPrincipal: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
dnAdminPassword: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
userBaseDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
groupBaseDN: ${AUTHENTICATION_GROUP_LOOKUP_BASEDN:-""}
|
||||
roleAdminName: ${AUTHENTICATION_USER_ROLE_ADMIN_NAME:-}
|
||||
allAttributeName: ${AUTHENTICATION_USER_ALL_ATTR:-}
|
||||
mailAttributeName: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
usernameAttributeName: ${AUTHENTICATION_USER_NAME_ATTR:-}
|
||||
groupAttributeName: ${AUTHENTICATION_USER_GROUP_ATTR:-}
|
||||
groupAttributeValue: ${AUTHENTICATION_USER_GROUP_ATTR_VALUE:-}
|
||||
groupMemberAttributeName: ${AUTHENTICATION_USER_GROUP_MEMBER_ATTR:-}
|
||||
#the mapping of roles to LDAP groups
|
||||
authRolesMapping: ${AUTH_ROLES_MAPPING:-""}
|
||||
authReassignRoles: ${AUTH_REASSIGN_ROLES:-[]}
|
||||
#optional
|
||||
maxPoolSize: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
sslEnabled: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
truststoreConfigType: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
trustStoreConfig:
|
||||
customTrustManagerConfig:
|
||||
trustStoreFilePath: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
trustStoreFilePassword: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
trustStoreFileFormat: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-}
|
||||
hostNameConfig:
|
||||
allowWildCards: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
acceptableHostNames: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
jvmDefaultConfig:
|
||||
verifyHostname: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
trustAllConfig:
|
||||
examineValidityDates: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
jwtTokenConfiguration:
|
||||
rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
|
||||
elasticsearch:
|
||||
searchType: ${SEARCH_TYPE:- "opensearch"}
|
||||
# Single host or comma-separated list for multiple hosts
|
||||
# Examples: "localhost" or "es-node1:9200,es-node2:9200,es-node3:9200"
|
||||
host: ${ELASTICSEARCH_HOST:-localhost}
|
||||
port: ${ELASTICSEARCH_PORT:-9200}
|
||||
scheme: ${ELASTICSEARCH_SCHEME:-http}
|
||||
username: ${ELASTICSEARCH_USER:-""}
|
||||
password: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
clusterAlias: ${ELASTICSEARCH_CLUSTER_ALIAS:-""}
|
||||
truststorePath: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-10} # Increased from 5s for Docker networks
|
||||
socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-120} # Increased from 60s for slow queries
|
||||
keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
# Connection pool settings for better load balancing and performance
|
||||
maxConnTotal: ${ELASTICSEARCH_MAX_CONN_TOTAL:-30} # Total connections across all hosts
|
||||
maxConnPerRoute: ${ELASTICSEARCH_MAX_CONN_PER_ROUTE:-10} # Max connections per host
|
||||
batchSize: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760}
|
||||
searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory
|
||||
# AWS IAM Authentication for OpenSearch (only applicable when searchType is "opensearch")
|
||||
# Uses standard AWS environment variables: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
# IAM auth is automatically enabled when AWS_DEFAULT_REGION is set
|
||||
# Credentials: Use AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or rely on AWS SDK default credential provider chain
|
||||
aws:
|
||||
enabled: ${SEARCH_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
serviceName: ${SEARCH_AWS_SERVICE_NAME:-"es"} # Use "es" for OpenSearch, "aoss" for OpenSearch Serverless
|
||||
naturalLanguageSearch:
|
||||
enabled: ${NATURAL_LANGUAGE_SEARCH_ENABLED:-false}
|
||||
embeddingProvider: ${EMBEDDING_PROVIDER:-bedrock}
|
||||
providerClass: ${NATURAL_LANGUAGE_SEARCH_PROVIDER_CLASS:-org.openmetadata.service.search.nlq.NoOpNLQService}
|
||||
bedrock:
|
||||
awsConfig:
|
||||
enabled: ${BEDROCK_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${AWS_DEFAULT_REGION:-""}
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
sessionToken: ${AWS_SESSION_TOKEN:-""}
|
||||
modelId: ${AWS_BEDROCK_MODEL_ID:-""}
|
||||
embeddingModelId: ${AWS_BEDROCK_EMBED_MODEL_ID:-""}
|
||||
embeddingDimension: ${AWS_BEDROCK_EMBEDDING_DIMENSION:-""}
|
||||
|
||||
eventMonitoringConfiguration:
|
||||
eventMonitor: ${EVENT_MONITOR:-prometheus} # Possible values are "prometheus", "cloudwatch"
|
||||
batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
|
||||
servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
|
||||
# it will use the default auth provider for AWS services if parameters are not set
|
||||
# parameters:
|
||||
# region: ${OM_MONITOR_REGION:-""}
|
||||
# accessKeyId: ${OM_MONITOR_ACCESS_KEY_ID:-""}
|
||||
# secretAccessKey: ${OM_MONITOR_ACCESS_KEY:-""}
|
||||
|
||||
eventHandlerConfiguration:
|
||||
eventHandlerClassNames:
|
||||
- "org.openmetadata.service.events.AuditEventHandler"
|
||||
- "org.openmetadata.service.events.ChangeEventHandler"
|
||||
|
||||
pipelineServiceClientConfiguration:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
# If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
|
||||
className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
|
||||
metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
|
||||
ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
# This SSL information is about the OpenMetadata server.
|
||||
# It will be picked up from the pipelineServiceClient to use/ignore SSL when connecting to the OpenMetadata server.
|
||||
verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
|
||||
sslConfig:
|
||||
certificatePath: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} # Local path for the Pipeline Service Client
|
||||
logStorageConfiguration:
|
||||
type: ${PIPELINE_SERVICE_CLIENT_LOG_TYPE:-"default"} # Possible values are "default", "s3"
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_LOG_ENABLED:-false} # Enable it for pipelines deployed in the server
|
||||
# if type is s3, provide the following configuration
|
||||
bucketName: ${PIPELINE_SERVICE_CLIENT_LOG_BUCKET_NAME:-""}
|
||||
# optional path within the bucket to store the logs
|
||||
prefix: ${PIPELINE_SERVICE_CLIENT_LOG_PREFIX:-""}
|
||||
enableServerSideEncryption: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ENABLED:-false}
|
||||
sseAlgorithm: ${PIPELINE_SERVICE_CLIENT_LOG_SSE_ALGORITHM:-"AES256"} # Allowed values: "AES256" or "aws:kms"
|
||||
kmsKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_KMS_KEY_ID:-""} # Required only if sseAlgorithm is "aws:kms"
|
||||
awsConfig:
|
||||
enabled: ${PIPELINE_SERVICE_CLIENT_AWS_IAM_AUTH_ENABLED:-false}
|
||||
awsAccessKeyId: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ACCESS_KEY_ID:-""}
|
||||
awsSecretAccessKey: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SECRET_ACCESS_KEY:-""}
|
||||
awsRegion: ${PIPELINE_SERVICE_CLIENT_LOG_REGION:-""}
|
||||
awsSessionToken: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_SESSION_TOKEN:-""}
|
||||
endPointURL: ${PIPELINE_SERVICE_CLIENT_LOG_AWS_ENDPOINT_URL:-""} # port forward localhost:9000 for minio
|
||||
|
||||
# Secrets Manager Loader: specify to the Ingestion Framework how to load the SM credentials from its env
|
||||
# Supported: noop, airflow, env
|
||||
secretsManagerLoader: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
|
||||
# Default required parameters for Airflow as Pipeline Service Client
|
||||
parameters:
|
||||
## Airflow parameters
|
||||
username: ${AIRFLOW_USERNAME:-admin}
|
||||
password: ${AIRFLOW_PASSWORD:-admin}
|
||||
timeout: ${AIRFLOW_TIMEOUT:-10}
|
||||
# If we need to use SSL to reach Airflow
|
||||
truststorePath: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
truststorePassword: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
## Kubernetes client parameters
|
||||
namespace: ${K8S_NAMESPACE:-"openmetadata-pipelines"}
|
||||
ingestionImage: ${K8S_INGESTION_IMAGE:-"docker.getcollate.io/openmetadata/ingestion-base:latest"}
|
||||
imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-"IfNotPresent"}
|
||||
imagePullSecrets: ${K8S_IMAGE_PULL_SECRETS:-""}
|
||||
serviceAccountName: ${K8S_SERVICE_ACCOUNT_NAME:-"openmetadata-ingestion"}
|
||||
# Resources configuration
|
||||
resources:
|
||||
limits:
|
||||
cpu: ${K8S_LIMITS_CPU:-"2"}
|
||||
memory: ${K8S_LIMITS_MEMORY:-"4Gi"}
|
||||
requests:
|
||||
cpu: ${K8S_REQUESTS_CPU:-"500m"}
|
||||
memory: ${K8S_REQUESTS_MEMORY:-"1Gi"}
|
||||
ttlSecondsAfterFinished: ${K8S_TTL_SECONDS_AFTER_FINISHED:-604800}
|
||||
activeDeadlineSeconds: ${K8S_ACTIVE_DEADLINE_SECONDS:-604800}
|
||||
backoffLimit: ${K8S_BACKOFF_LIMIT:-3}
|
||||
successfulJobsHistoryLimit: ${K8S_SUCCESSFUL_JOBS_HISTORY_LIMIT:-3}
|
||||
failedJobsHistoryLimit: ${K8S_FAILED_JOBS_HISTORY_LIMIT:-1}
|
||||
nodeSelector: ${K8S_NODE_SELECTOR:-""}
|
||||
runAsUser: ${K8S_RUN_AS_USER:-1000}
|
||||
runAsGroup: ${K8S_RUN_AS_GROUP:-1000}
|
||||
fsGroup: ${K8S_FS_GROUP:-1000}
|
||||
runAsNonRoot: ${K8S_RUN_AS_NON_ROOT:-"true"}
|
||||
extraEnvVars: ${K8S_EXTRA_ENV_VARS:-[]}
|
||||
podAnnotations: ${K8S_POD_ANNOTATIONS:-""}
|
||||
useOMJobOperator: ${USE_OMJOB_OPERATOR:-"true"}
|
||||
|
||||
|
||||
# no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
|
||||
# to secure your instance of OpenMetadata with TLS and encryption at rest.
|
||||
fernetConfiguration:
|
||||
fernetKey: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
secretsManagerConfiguration:
|
||||
secretsManager: ${SECRET_MANAGER:-db} # Possible values are "db", "managed-aws","aws", "managed-aws-ssm", "aws-ssm", "managed-azure-kv", "azure-kv", "in-memory", "gcp", "kubernetes"
|
||||
prefix: ${SECRET_MANAGER_PREFIX:-""} # Define the secret key ID as /<prefix>/<clusterName>/<key>
|
||||
tags: ${SECRET_MANAGER_TAGS:-[]} # Add tags to the created resource. Format is `[key1:value1,key2:value2,...]`
|
||||
# it will use the default auth provider for the secrets' manager service if parameters are not set
|
||||
parameters:
|
||||
## For AWS
|
||||
accessKeyId: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${OM_SM_ACCESS_KEY:-""}
|
||||
## For Azure Key Vault
|
||||
clientId: ${OM_SM_CLIENT_ID:-""}
|
||||
clientSecret: ${OM_SM_CLIENT_SECRET:-""}
|
||||
tenantId: ${OM_SM_TENANT_ID:-""}
|
||||
vaultName: ${OM_SM_VAULT_NAME:-""}
|
||||
## For GCP
|
||||
projectId: ${OM_SM_PROJECT_ID:-""}
|
||||
## For Kubernetes
|
||||
namespace: ${OM_SM_NAMESPACE:-"default"}
|
||||
kubeconfigPath: ${OM_SM_KUBECONFIG_PATH:-""}
|
||||
inCluster: ${OM_SM_IN_CLUSTER:-"false"}
|
||||
|
||||
health:
|
||||
delayedShutdownHandlerEnabled: true
|
||||
shutdownWaitPeriod: 1s
|
||||
healthChecks:
|
||||
- name: OpenMetadataServerHealthCheck
|
||||
critical: true
|
||||
schedule:
|
||||
checkInterval: 2500ms
|
||||
downtimeInterval: 10s
|
||||
failureAttempts: 2
|
||||
successAttempts: 1
|
||||
|
||||
limits:
|
||||
enable: ${LIMITS_ENABLED:-false}
|
||||
className: ${LIMITS_CLASS_NAME:-"org.openmetadata.service.limits.DefaultLimits"}
|
||||
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}
|
||||
|
||||
# Bulk Operation Configuration
|
||||
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
|
||||
# Uses a bounded thread pool to prevent connection pool exhaustion
|
||||
bulkOperation:
|
||||
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
|
||||
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
|
||||
# Max queued operations before rejection (returns 503)
|
||||
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
|
||||
# Timeout in seconds for entire bulk operation
|
||||
timeoutSeconds: ${BULK_OPERATION_TIMEOUT_SECONDS:-300}
|
||||
|
||||
web:
|
||||
uriPath: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
hsts:
|
||||
enabled: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
maxAge: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
includeSubDomains: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
preload: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
frame-options:
|
||||
enabled: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
option: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
origin: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
content-type-options:
|
||||
enabled: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
xss-protection:
|
||||
enabled: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
on: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
block: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
csp:
|
||||
enabled: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
policy: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
reportOnlyPolicy: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
referrer-policy:
|
||||
enabled: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
permission-policy:
|
||||
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
pragma: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
operationalConfig:
|
||||
enable: ${OPERATIONAL_CONFIG_ENABLED:-true}
|
||||
operationsConfigFile: ${OPERATIONAL_CONFIG_FILE:-"./conf/operations.yaml"}
|
||||
|
||||
rdf:
|
||||
enabled: ${RDF_ENABLED:-false}
|
||||
baseUri: ${RDF_BASE_URI:-"https://open-metadata.org/"}
|
||||
storageType: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
remoteEndpoint: ${RDF_ENDPOINT:-"http://localhost:3030/openmetadata"}
|
||||
username: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
password: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
dataset: ${RDF_DATASET:-"openmetadata"}
|
||||
|
||||
# Cache Configuration
|
||||
# Caching layer for entity metadata, relationships, and tag usage to reduce database load
|
||||
# Default: Disabled (uses NoopCacheProvider)
|
||||
cache:
|
||||
# Cache provider: none (default) or redis
|
||||
provider: ${CACHE_PROVIDER:-none}
|
||||
|
||||
# TTL (Time To Live) settings in seconds
|
||||
entityTtlSeconds: ${CACHE_ENTITY_TTL:-172800} # 48 hour for entities
|
||||
relationshipTtlSeconds: ${CACHE_RELATIONSHIP_TTL:-172800} # 48 hour for relationships
|
||||
tagTtlSeconds: ${CACHE_TAG_TTL:-172800} # 48 hour for tags
|
||||
|
||||
# Redis configuration
|
||||
redis:
|
||||
# Redis connection URL
|
||||
# Standalone: redis://localhost:6379
|
||||
# AWS ElastiCache: redis://my-cluster.abc123.cache.amazonaws.com:6379
|
||||
url: ${CACHE_REDIS_URL:-redis://localhost:6379}
|
||||
authType: ${CACHE_REDIS_AUTH_TYPE:-NONE}
|
||||
database: ${CACHE_REDIS_DATABASE:-0} # Redis database index (0-15)
|
||||
|
||||
# Authentication for standalone Redis
|
||||
username: ${CACHE_REDIS_USERNAME:-}
|
||||
passwordRef: ${CACHE_REDIS_PASSWORD:-} # Reference to password in secrets manager
|
||||
useSSL: ${CACHE_REDIS_USE_SSL:-false}
|
||||
|
||||
# Key namespace prefix (useful for multi-tenant deployments)
|
||||
keyspace: ${CACHE_REDIS_KEYSPACE:-"om:prod"}
|
||||
|
||||
# Connection pool settings
|
||||
poolSize: ${CACHE_REDIS_POOL_SIZE:-64}
|
||||
connectTimeoutMs: ${CACHE_REDIS_CONNECT_TIMEOUT:-2000}
|
||||
|
||||
# AWS ElastiCache IAM Authentication (only if using ElastiCache)
|
||||
aws:
|
||||
enabled: ${CACHE_REDIS_AWS_IAM_AUTH_ENABLED:-false}
|
||||
region: ${CACHE_REDIS_AWS_REGION:-""}
|
||||
useInstanceProfile: ${CACHE_REDIS_AWS_INSTANCE_PROFILE:-true}
|
||||
# If not using instance profile, provide credentials:
|
||||
accessKeyId: ${AWS_ACCESS_KEY_ID:-""}
|
||||
secretAccessKey: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
tokenRefreshIntervalSeconds: ${CACHE_REDIS_TOKEN_REFRESH:-900} # 15 minutes
|
||||
@@ -0,0 +1,96 @@
|
||||
version: "3.9"
|
||||
|
||||
# Compose override for RDF-enabled local stacks.
|
||||
# Use together with docker-compose.yml or docker-compose-postgres.yml.
|
||||
services:
|
||||
execute-migrate-all:
|
||||
environment:
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_DATASET: ${RDF_DATASET:-openmetadata}
|
||||
depends_on:
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
|
||||
openmetadata-server:
|
||||
environment:
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_DATASET: ${RDF_DATASET:-openmetadata}
|
||||
depends_on:
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
|
||||
fuseki:
|
||||
# Build from the in-repo Dockerfile (Fuseki 5.6.0) instead of the
|
||||
# unmaintained `stain/jena-fuseki` Docker Hub image, which capped at 5.1.0
|
||||
# and never picked up the 2025 admin-side Fuseki CVE fixes (CVE-2025-49656,
|
||||
# CVE-2025-50151 — both fixed in Jena 5.5.0). The `image:` tag below names
|
||||
# the locally-built image so subsequent `docker compose up` runs reuse the
|
||||
# cached build instead of rebuilding from scratch each time.
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: openmetadata-fuseki
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
networks:
|
||||
- local_app_net
|
||||
environment:
|
||||
# Default for local dev only — production deployments MUST override
|
||||
# via the FUSEKI_ADMIN_PASSWORD env var (and FUSEKI_OPENMETADATA_PASSWORD)
|
||||
# before bringing this stack up. The entrypoint envsubsts these into
|
||||
# shiro.ini at container start so the override actually takes effect.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=${FUSEKI_JVM_ARGS:--Xmx1500m -Xms256m}
|
||||
volumes:
|
||||
# New volume name (was `fuseki-data` mounted at `/fuseki`). The in-repo
|
||||
# Dockerfile stores TDB2 at `/fuseki-data` and the data layout differs
|
||||
# from the old stain/jena-fuseki image — re-using the previous volume
|
||||
# name would mount stale state at a path Fuseki no longer reads from,
|
||||
# silently looking like an empty database. Using a fresh volume name
|
||||
# forces operators to consciously migrate (or accept a re-index). The
|
||||
# orphaned `fuseki-data` volume can be removed manually with
|
||||
# `docker volume rm fuseki-data` after confirming the new stack is
|
||||
# healthy.
|
||||
- fuseki-tdb2-data:/fuseki-data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
reservations:
|
||||
memory: 256m
|
||||
restart: "on-failure:3"
|
||||
healthcheck:
|
||||
test: "curl -s -f http://localhost:3030/\\$/ping > /dev/null || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 60s
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
|
||||
volumes:
|
||||
fuseki-tdb2-data:
|
||||
driver: local
|
||||
@@ -0,0 +1,571 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
###
|
||||
# How to use this compose yaml file.
|
||||
# You have to prepare the credential json file for the service account which can access the GCP secret manager and set the path of the json file into the GOOGLE_APPLICATION_CREDENTIALS variable.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
mysql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/mysql/Dockerfile_mysql
|
||||
command: "--sort_buffer_size=10M"
|
||||
container_name: openmetadata_mysql
|
||||
restart: always
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
container_name: openmetadata_elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
# HACK: This is hack for M1 mac or later to avoid aborting JVM by the Google Secret Manager library.
|
||||
platform: linux/amd64
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: /key.json
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
# GCP:
|
||||
OM_SM_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
volumes:
|
||||
- ${GOOGLE_APPLICATION_CREDENTIALS}:/key.json:ro
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: /key.json
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
|
||||
## Secrets Manager
|
||||
# To integrate Azure Key Vault
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_KEY_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
# To Integrate with AWS SSM
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_REGION: ${OM_SM_AWS_REGION:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY_ID: ${OM_SM_AWS_ACCESS_KEY_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_SECRET_ACCESS_KEY: ${OM_SM_AWS_SECRET_ACCESS_KEY:-""}
|
||||
# To integrate GCP
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ${GOOGLE_APPLICATION_CREDENTIALS}:/key.json:ro
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,614 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
# See docker-compose-fuseki.yml — renamed from `fuseki-data` to avoid
|
||||
# silently mounting stale state under the new Fuseki layout.
|
||||
fuseki-tdb2-data:
|
||||
services:
|
||||
postgresql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
container_name: openmetadata_postgresql
|
||||
restart: always
|
||||
command: "--work_mem=10MB"
|
||||
depends_on:
|
||||
- opensearch
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: password
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: openmetadata_opensearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- plugins.security.disabled=true
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!!
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/opensearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for Postgres
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-admin}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
fuseki:
|
||||
condition: service_started
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_started
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
fuseki:
|
||||
# See docker-compose-fuseki.yml for the rationale behind building from the
|
||||
# in-repo Dockerfile instead of using `stain/jena-fuseki:*` (unmaintained,
|
||||
# capped at 5.1.0, missing 2025 admin-side CVE fixes).
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: openmetadata-fuseki
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Local-dev default — production deployments MUST override via
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
volumes:
|
||||
# See docker-compose-fuseki.yml for why the volume was renamed from
|
||||
# `fuseki-data` to `fuseki-tdb2-data` (the data layout differs from the
|
||||
# previous stain/jena-fuseki image and reusing the old name silently
|
||||
# mounts stale state at a path Fuseki no longer reads).
|
||||
- fuseki-tdb2-data:/fuseki-data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,569 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
postgresql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
container_name: openmetadata_postgresql
|
||||
restart: always
|
||||
command: "--work_mem=10MB"
|
||||
depends_on:
|
||||
- opensearch
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: password
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
container_name: openmetadata_opensearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- plugins.security.disabled=true
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!!
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/opensearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
# Application connector type. Set SERVER_PROTOCOL=h2c on the host to enable
|
||||
# cleartext HTTP/2 at Jetty. Admin connector stays HTTP/1.1 regardless.
|
||||
SERVER_PROTOCOL: ${SERVER_PROTOCOL:-http}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for Postgres
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-opensearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_started
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,8 @@
|
||||
# Override that disables the cache while leaving the rest of the stack intact.
|
||||
# Used in the local A/B benchmark to flip cache off without tearing down volumes.
|
||||
# Apply on TOP of base compose (NOT the redis overlay):
|
||||
# docker compose -f docker-compose.yml -f docker-compose.cache-off.yml up -d --no-deps openmetadata-server
|
||||
services:
|
||||
openmetadata-server:
|
||||
environment:
|
||||
CACHE_PROVIDER: none
|
||||
@@ -0,0 +1,86 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
# Adds a second OM instance that shares MySQL/Elasticsearch/Redis with the
|
||||
# primary one in docker-compose.yml. Used to validate that pub/sub
|
||||
# invalidation keeps per-instance Guava caches coherent.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.redis.yml \
|
||||
# -f docker-compose.multiserver.yml up -d
|
||||
services:
|
||||
openmetadata-server-2:
|
||||
image: development-openmetadata-server
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server_2
|
||||
restart: always
|
||||
networks:
|
||||
- local_app_net
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8587:8585"
|
||||
- "8588:8586"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: openmetadata
|
||||
SERVER_PORT: 8585
|
||||
SERVER_ADMIN_PORT: 8586
|
||||
LOG_LEVEL: INFO
|
||||
FERNET_KEY: jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=
|
||||
DB_DRIVER_CLASS: com.mysql.cj.jdbc.Driver
|
||||
DB_SCHEME: mysql
|
||||
DB_USE_SSL: "false"
|
||||
DB_USER: openmetadata_user
|
||||
DB_USER_PASSWORD: openmetadata_password
|
||||
DB_HOST: mysql
|
||||
DB_PORT: 3306
|
||||
DB_PARAMS: allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
|
||||
OM_DATABASE: openmetadata_db
|
||||
ELASTICSEARCH_HOST: elasticsearch
|
||||
ELASTICSEARCH_PORT: 9200
|
||||
ELASTICSEARCH_SCHEME: http
|
||||
SEARCH_TYPE: elasticsearch
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: openmetadata
|
||||
AUTHENTICATION_PROVIDER: basic
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: "true"
|
||||
AUTHORIZER_CLASS_NAME: org.openmetadata.service.security.DefaultAuthorizer
|
||||
AUTHORIZER_REQUEST_FILTER: org.openmetadata.service.security.JwtFilter
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: "[admin]"
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: open-metadata.org
|
||||
AUTHORIZER_ALLOWED_DOMAINS: "[]"
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: '["all"]'
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: "[ingestion-bot]"
|
||||
AUTHENTICATION_RESPONSE_TYPE: id_token
|
||||
AUTHENTICATION_CLIENT_TYPE: public
|
||||
AUTHENTICATION_PUBLIC_KEYS: "[http://openmetadata-server-2:8585/api/v1/system/config/jwks]"
|
||||
AUTHENTICATION_AUTHORITY: https://accounts.google.com
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: "[email,preferred_username,sub]"
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ./conf/public_key.der
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ./conf/private_key.der
|
||||
JWT_ISSUER: open-metadata.org
|
||||
JWT_KEY_ID: Gb389a-9f76-gdjs-a92j-0242bk94356
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: http://ingestion:8080
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient
|
||||
AIRFLOW_USERNAME: admin
|
||||
AIRFLOW_PASSWORD: admin
|
||||
AIRFLOW_TIMEOUT: 10
|
||||
SECRET_MANAGER: db
|
||||
SERVER_HOST_API_URL: http://openmetadata-server-2:8585/api
|
||||
EVENT_MONITOR: prometheus
|
||||
OPENMETADATA_HEAP_OPTS: "-Xmx1G -Xms1G"
|
||||
CACHE_PROVIDER: redis
|
||||
CACHE_REDIS_URL: redis://redis:6379
|
||||
CACHE_REDIS_AUTH_TYPE: NONE
|
||||
CACHE_REDIS_KEYSPACE: om:dev
|
||||
CACHE_ENTITY_TTL: 3600
|
||||
CACHE_RELATIONSHIP_TTL: 3600
|
||||
CACHE_TAG_TTL: 3600
|
||||
CACHE_REDIS_COMMAND_TIMEOUT: 300
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
# Override that adds a Redis cache to the development stack.
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.redis.yml up -d
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: openmetadata_redis
|
||||
restart: always
|
||||
command: ["redis-server", "--appendonly", "no", "--save", "", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
networks:
|
||||
- local_app_net
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
openmetadata-server:
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
CACHE_PROVIDER: redis
|
||||
CACHE_REDIS_URL: redis://redis:6379
|
||||
CACHE_REDIS_AUTH_TYPE: NONE
|
||||
CACHE_REDIS_KEYSPACE: om:dev
|
||||
CACHE_ENTITY_TTL: 3600
|
||||
CACHE_RELATIONSHIP_TTL: 3600
|
||||
CACHE_TAG_TTL: 3600
|
||||
CACHE_REDIS_COMMAND_TIMEOUT: 300
|
||||
@@ -0,0 +1,652 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
fuseki-data:
|
||||
services:
|
||||
mysql:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/mysql/Dockerfile_mysql
|
||||
command: "--sort_buffer_size=10M"
|
||||
container_name: openmetadata_mysql
|
||||
restart: always
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
container_name: openmetadata_elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=${ES_JAVA_OPTS:--Xms1024m -Xmx1024m}
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 9200
|
||||
- 9300
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# RDF / SPARQL backend (optional; default off). Start fuseki with `--profile rdf`.
|
||||
RDF_ENABLED: ${RDF_ENABLED:-false}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-"http://fuseki:3030/openmetadata"}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
RDF_DATASET: ${RDF_DATASET:-"openmetadata"}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- local_app_net
|
||||
|
||||
openmetadata-server:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: docker/development/Dockerfile
|
||||
container_name: openmetadata_server
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
# Application connector type. Set SERVER_PROTOCOL=h2c on the host to enable
|
||||
# cleartext HTTP/2 at Jetty. Admin connector stays HTTP/1.1 regardless.
|
||||
SERVER_PROTOCOL: ${SERVER_PROTOCOL:-http}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# RDF / SPARQL backend (optional; default off). Start fuseki with `--profile rdf`.
|
||||
RDF_ENABLED: ${RDF_ENABLED:-false}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-"FUSEKI"}
|
||||
RDF_ENDPOINT: ${RDF_ENDPOINT:-"http://fuseki:3030/openmetadata"}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-"admin"}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-"admin"}
|
||||
RDF_DATASET: ${RDF_DATASET:-"openmetadata"}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_CLUSTER_ALIAS: ${ELASTICSEARCH_CLUSTER_ALIAS:- "openmetadata"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
# GCP:
|
||||
OM_SM_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-true}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
- 5005
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
build:
|
||||
context: ../../.
|
||||
dockerfile: ingestion/Dockerfile.ci
|
||||
args:
|
||||
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
|
||||
container_name: openmetadata_ingestion
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__LOGGING__LOGGING_LEVEL: ${AIRFLOW_LOGGING_LEVEL:-DEBUG}
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
# OpenLineage transport config (optional - enable for lineage via OL)
|
||||
# AIRFLOW__OPENLINEAGE__TRANSPORT: '{"type": "http", "url": "http://openmetadata-server:8585/api/v1/openlineage/", "endpoint": "lineage", "auth": {"type": "api_key", "api_key": "<OM_JWT_TOKEN>"}}'
|
||||
# AIRFLOW__OPENLINEAGE__NAMESPACE: local_airflow
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
# AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
||||
# AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: true
|
||||
|
||||
## Secrets Manager
|
||||
# To integrate Azure Key Vault
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_KEY_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AZURE_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
# To Integrate with AWS SSM
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_REGION: ${OM_SM_AWS_REGION:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY_ID: ${OM_SM_AWS_ACCESS_KEY_ID:-""}
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_SECRET_ACCESS_KEY: ${OM_SM_AWS_SECRET_ACCESS_KEY:-""}
|
||||
# To integrate GCP
|
||||
AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
|
||||
|
||||
# Apps
|
||||
ENABLE_APP_HelloPipelines: "true"
|
||||
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- local_app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
- /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator
|
||||
|
||||
|
||||
mock-oidc-provider:
|
||||
build:
|
||||
context: ./mock-oidc-provider
|
||||
dockerfile: Dockerfile
|
||||
container_name: mock_oidc_provider
|
||||
environment:
|
||||
PORT: "9090"
|
||||
ISSUER: ${MOCK_OIDC_ISSUER:-http://localhost:9090}
|
||||
INTERNAL_BASE_URL: ${MOCK_OIDC_INTERNAL_BASE_URL:-http://mock-oidc-provider:9090}
|
||||
expose:
|
||||
- 9090
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks:
|
||||
- local_app_net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
profiles:
|
||||
- sso-test
|
||||
|
||||
# Apache Jena Fuseki triplestore for the optional RDF/SPARQL backend (ontology
|
||||
# round-trip, semantic search). Start with `--profile rdf` and set RDF_ENABLED=true.
|
||||
# The glossary RDF import works without this; it only adds the transparent triplestore.
|
||||
fuseki:
|
||||
build:
|
||||
context: ../rdf-store
|
||||
container_name: openmetadata-fuseki
|
||||
environment:
|
||||
# The admin user carries the writer role; the server connects as
|
||||
# RDF_REMOTE_USERNAME=admin, so keep this password in sync with
|
||||
# RDF_REMOTE_PASSWORD. Dataset (/openmetadata) is baked into the image.
|
||||
- FUSEKI_ADMIN_PASSWORD=${RDF_REMOTE_PASSWORD:-admin}
|
||||
networks:
|
||||
- local_app_net
|
||||
expose:
|
||||
- 3030
|
||||
ports:
|
||||
- "3030:3030"
|
||||
volumes:
|
||||
- fuseki-data:/fuseki-data
|
||||
profiles:
|
||||
- rdf
|
||||
|
||||
networks:
|
||||
local_app_net:
|
||||
name: ometa_network
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.239.0/24"
|
||||
@@ -0,0 +1,295 @@
|
||||
# OpenMetadata Helm Chart Local Testing
|
||||
|
||||
Helper to test changes from https://github.com/open-metadata/openmetadata-helm-charts with local images while developing.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Required Tools
|
||||
```bash
|
||||
# Install required tools
|
||||
brew install helm kubectl minikube docker-compose
|
||||
|
||||
# Or on Ubuntu/Debian:
|
||||
# sudo snap install helm kubectl minikube
|
||||
# sudo apt-get install docker-compose
|
||||
```
|
||||
|
||||
### Required Resources
|
||||
- **CPU**: 4+ cores recommended
|
||||
- **Memory**: 8GB+ RAM recommended
|
||||
- **Storage**: 20GB+ free space
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Start Local Dependencies
|
||||
|
||||
First, start the required database and search services:
|
||||
|
||||
```bash
|
||||
cd docker/development/helm
|
||||
|
||||
# Start PostgreSQL and OpenSearch
|
||||
docker-compose -f docker-compose-deps.yml up -d
|
||||
|
||||
# Wait for services to be ready (2-3 minutes)
|
||||
docker-compose -f docker-compose-deps.yml logs -f
|
||||
|
||||
# Verify services are running
|
||||
curl http://localhost:9200/_cluster/health
|
||||
docker exec openmetadata_postgres_test psql -U openmetadata_user -d openmetadata_db -c "SELECT 1"
|
||||
```
|
||||
|
||||
### 2. Start Kubernetes Cluster
|
||||
|
||||
```bash
|
||||
# Start minikube with sufficient resources
|
||||
minikube start --cpus 4 --memory 8192 --driver docker
|
||||
|
||||
# Enable ingress addon (optional)
|
||||
minikube addons enable ingress
|
||||
|
||||
# Verify cluster is ready
|
||||
kubectl cluster-info
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
### 3. Create Required Secrets
|
||||
|
||||
```bash
|
||||
# Create secrets that the chart expects
|
||||
kubectl create secret generic postgres-secrets \
|
||||
--from-literal=openmetadata-postgres-password=openmetadata_password
|
||||
|
||||
kubectl create secret generic airflow-secrets \
|
||||
--from-literal=openmetadata-airflow-password=admin
|
||||
```
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Scenario A: Test Kubernetes Native Pipeline Client (NEW)
|
||||
|
||||
Test the new K8s native pipeline execution without Airflow dependencies.
|
||||
|
||||
```bash
|
||||
# Use local chart directly, e.g.,
|
||||
CHART_PATH="/Users/pmbrull/github/openmetadata-helm-charts/charts/openmetadata"
|
||||
|
||||
# Install with K8s native configuration
|
||||
helm install openmetadata-k8s-test $CHART_PATH --values values-k8s-test.yaml
|
||||
|
||||
# Check deployment status
|
||||
kubectl get pods -A
|
||||
kubectl get jobs -n openmetadata-pipelines-test
|
||||
kubectl get serviceaccounts,roles,rolebindings -n openmetadata-pipelines-test
|
||||
|
||||
# Check logs
|
||||
kubectl logs -l app.kubernetes.io/name=openmetadata -f
|
||||
|
||||
# Test access
|
||||
minikube tunnel # In separate terminal
|
||||
curl http://localhost:8585/api/v1/system/health
|
||||
```
|
||||
|
||||
### Scenario B: Test Migrated Airflow Configuration
|
||||
|
||||
Test that the new nested Airflow configuration still works.
|
||||
|
||||
```bash
|
||||
# Install with migrated Airflow configuration
|
||||
helm install openmetadata-airflow-test $CHART_PATH \
|
||||
--values values-airflow-test.yaml \
|
||||
--timeout 10m \
|
||||
--wait
|
||||
|
||||
# Check deployment
|
||||
kubectl get pods -A
|
||||
kubectl logs -l app.kubernetes.io/name=openmetadata -f
|
||||
```
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
### ✅ Basic Functionality
|
||||
|
||||
1. **Pod Health**
|
||||
```bash
|
||||
# Check all pods are running
|
||||
kubectl get pods -A
|
||||
|
||||
# Check OpenMetadata pod logs
|
||||
kubectl logs deployment/openmetadata-k8s-test -f
|
||||
|
||||
# Check database connectivity
|
||||
kubectl exec deployment/openmetadata-k8s-test -- curl -f http://postgres:5432 || echo "DB connection test"
|
||||
```
|
||||
|
||||
2. **API Health**
|
||||
```bash
|
||||
# Access health endpoint
|
||||
curl http://localhost:8585/api/v1/system/health
|
||||
|
||||
# Check API response
|
||||
curl http://localhost:8585/api/v1/system/config
|
||||
```
|
||||
|
||||
### ✅ K8s Native Pipeline Features
|
||||
|
||||
1. **RBAC Resources**
|
||||
```bash
|
||||
# Check namespace creation
|
||||
kubectl get namespace openmetadata-pipelines-test
|
||||
|
||||
# Check service account
|
||||
kubectl get serviceaccount -n openmetadata-pipelines-test openmetadata-ingestion-test
|
||||
|
||||
# Check permissions
|
||||
kubectl auth can-i create jobs \
|
||||
--as=system:serviceaccount:openmetadata-pipelines-test:openmetadata-ingestion-test \
|
||||
-n openmetadata-pipelines-test
|
||||
```
|
||||
|
||||
2. **Configuration Validation**
|
||||
```bash
|
||||
# Check environment variables in pod
|
||||
kubectl exec deployment/openmetadata-k8s-test -- env | grep K8S_
|
||||
|
||||
# Check secrets
|
||||
kubectl get secret openmetadata-k8s-test-pipeline-secret -o yaml
|
||||
kubectl get secret openmetadata-k8s-test-pipeline-secret -o jsonpath='{.data}' | base64 -d
|
||||
```
|
||||
|
||||
3. **Pipeline Job Testing**
|
||||
```bash
|
||||
# Create a test pipeline job (manual)
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: test-ingestion-job
|
||||
namespace: openmetadata-pipelines-test
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: openmetadata-ingestion-test
|
||||
containers:
|
||||
- name: test
|
||||
image: docker.getcollate.io/openmetadata/ingestion:latest
|
||||
command: ["echo", "Test pipeline job works"]
|
||||
restartPolicy: Never
|
||||
EOF
|
||||
|
||||
# Check job execution
|
||||
kubectl get jobs -n openmetadata-pipelines-test
|
||||
kubectl logs job/test-ingestion-job -n openmetadata-pipelines-test
|
||||
```
|
||||
|
||||
### ✅ Failure Diagnostics Testing
|
||||
|
||||
1. **Create Failing Job**
|
||||
```bash
|
||||
# Create a job that will fail
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: test-failing-job
|
||||
namespace: openmetadata-pipelines-test
|
||||
labels:
|
||||
app.kubernetes.io/pipeline: test-pipeline
|
||||
app.kubernetes.io/run-id: test-123
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: openmetadata-ingestion-test
|
||||
containers:
|
||||
- name: main
|
||||
image: docker.getcollate.io/openmetadata/ingestion:latest
|
||||
command: ["sh", "-c", "echo 'Starting ingestion...'; sleep 10; echo 'Something went wrong!'; exit 1"]
|
||||
restartPolicy: Never
|
||||
EOF
|
||||
|
||||
# Watch for diagnostic job creation
|
||||
kubectl get jobs -n openmetadata-pipelines-test -w
|
||||
|
||||
# Check diagnostic job logs
|
||||
kubectl logs -n openmetadata-pipelines-test -l app.kubernetes.io/component=diagnostics
|
||||
```
|
||||
|
||||
### ✅ Configuration Migration Testing
|
||||
|
||||
1. **Test Both Configurations**
|
||||
```bash
|
||||
# Test that both airflow and k8s configs are valid
|
||||
helm template test-airflow $CHART_PATH --values values-airflow-test.yaml > /tmp/airflow-manifest.yaml
|
||||
helm template test-k8s $CHART_PATH --values values-k8s-test.yaml > /tmp/k8s-manifest.yaml
|
||||
|
||||
# Validate manifests
|
||||
kubectl apply --dry-run=client -f /tmp/airflow-manifest.yaml
|
||||
kubectl apply --dry-run=client -f /tmp/k8s-manifest.yaml
|
||||
```
|
||||
|
||||
2. **Test Breaking Changes**
|
||||
```bash
|
||||
# Test old configuration format (should fail validation or show warnings)
|
||||
cat > /tmp/old-values.yaml << EOF
|
||||
openmetadata:
|
||||
config:
|
||||
pipelineServiceClientConfig:
|
||||
enabled: true
|
||||
className: "org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"
|
||||
apiEndpoint: http://test
|
||||
EOF
|
||||
|
||||
helm template test-old $CHART_PATH --values /tmp/old-values.yaml
|
||||
```
|
||||
|
||||
### Debug Commands
|
||||
|
||||
```bash
|
||||
# Get all resources
|
||||
kubectl get all -A
|
||||
|
||||
# Describe OpenMetadata deployment
|
||||
kubectl describe deployment openmetadata-k8s-test
|
||||
|
||||
# Get events
|
||||
kubectl get events --sort-by='.lastTimestamp' -A
|
||||
|
||||
# Check helm release
|
||||
helm list
|
||||
helm status openmetadata-k8s-test
|
||||
helm get values openmetadata-k8s-test
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
### Remove Test Deployments
|
||||
```bash
|
||||
# Remove helm releases
|
||||
helm uninstall openmetadata-k8s-test
|
||||
helm uninstall openmetadata-airflow-test
|
||||
|
||||
# Clean up namespaces
|
||||
kubectl delete namespace openmetadata-pipelines-test
|
||||
|
||||
# Clean up secrets
|
||||
kubectl delete secret postgres-secrets airflow-secrets
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
### Successful Deployment Indicators
|
||||
|
||||
1. ✅ **All pods running**: `kubectl get pods -A` shows all pods in `Running` state
|
||||
2. ✅ **Health check passing**: `curl http://localhost:8585/api/v1/system/health` returns 200
|
||||
3. ✅ **RBAC created**: K8s resources created in `openmetadata-pipelines-test` namespace
|
||||
4. ✅ **Configuration loaded**: Environment variables properly set in pods
|
||||
5. ✅ **No errors in logs**: `kubectl logs deployment/openmetadata-k8s-test` shows successful startup
|
||||
|
||||
### Performance Benchmarks
|
||||
|
||||
- **Startup time**: < 5 minutes for full deployment
|
||||
- **Memory usage**: < 4GB total (including dependencies)
|
||||
- **CPU usage**: < 2 cores under normal load
|
||||
- **Job creation**: < 30 seconds for pipeline job creation and execution
|
||||
|
||||
This comprehensive testing suite validates both the new K8s native functionality and ensures backward compatibility with existing Airflow configurations.
|
||||
@@ -0,0 +1,82 @@
|
||||
# Docker compose for OpenMetadata dependencies (PostgreSQL + OpenSearch)
|
||||
# Use this for local Helm chart testing
|
||||
|
||||
version: "3.9"
|
||||
services:
|
||||
|
||||
# PostgreSQL Database
|
||||
postgres:
|
||||
container_name: openmetadata_postgres_test
|
||||
build:
|
||||
context: ../../../
|
||||
dockerfile: docker/postgresql/Dockerfile_postgres
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_DB: openmetadata_db
|
||||
POSTGRES_USER: openmetadata_user
|
||||
POSTGRES_PASSWORD: openmetadata_password
|
||||
POSTGRES_ROOT_PASSWORD: password
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5433:5432" # Use different port to avoid conflicts
|
||||
networks:
|
||||
- openmetadata_network
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
command: >
|
||||
postgres
|
||||
-c shared_preload_libraries=pg_stat_statements
|
||||
-c max_connections=200
|
||||
-c shared_buffers=256MB
|
||||
-c effective_cache_size=1GB
|
||||
-c maintenance_work_mem=64MB
|
||||
-c checkpoint_completion_target=0.9
|
||||
-c wal_buffers=16MB
|
||||
-c default_statistics_target=100
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openmetadata_user -d openmetadata_db"]
|
||||
timeout: 20s
|
||||
retries: 10
|
||||
interval: 30s
|
||||
|
||||
# OpenSearch for metadata search
|
||||
opensearch:
|
||||
container_name: openmetadata_opensearch_test
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
restart: always
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- node.name=opensearch
|
||||
- cluster.name=opensearch-cluster
|
||||
- bootstrap.memory_lock=true
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m"
|
||||
- "DISABLE_INSTALL_DEMO_CONFIG=true"
|
||||
- "DISABLE_SECURITY_PLUGIN=true"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
networks:
|
||||
- openmetadata_network
|
||||
volumes:
|
||||
- opensearch_data:/usr/share/opensearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
opensearch_data:
|
||||
|
||||
networks:
|
||||
openmetadata_network:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,67 @@
|
||||
# Test values for migrated Airflow pipeline client validation
|
||||
# Only contains essential overrides for testing the new nested structure
|
||||
|
||||
openmetadata:
|
||||
config:
|
||||
# Enable debug logging for testing
|
||||
logLevel: DEBUG
|
||||
|
||||
# Database configuration (using local PostgreSQL via host machine)
|
||||
database:
|
||||
host: host.docker.internal
|
||||
port: 5433
|
||||
driverClass: org.postgresql.Driver
|
||||
dbScheme: postgresql
|
||||
auth:
|
||||
password:
|
||||
secretRef: postgres-secrets
|
||||
secretKey: openmetadata-postgres-password
|
||||
|
||||
# Search configuration (using local OpenSearch via host machine)
|
||||
elasticsearch:
|
||||
host: host.docker.internal
|
||||
port: 9200
|
||||
searchType: opensearch
|
||||
scheme: http
|
||||
|
||||
# Airflow Pipeline Client Configuration (updated structure with common configs)
|
||||
pipelineServiceClientConfig:
|
||||
type: "airflow" # Use traditional Airflow approach
|
||||
# Common configuration for all pipeline service clients
|
||||
metadataApiEndpoint: http://openmetadata:8585/api
|
||||
airflow:
|
||||
# Airflow-specific connection settings
|
||||
apiEndpoint: http://openmetadata-dependencies-api-server:8080
|
||||
verifySsl: "no-ssl"
|
||||
auth:
|
||||
username: admin
|
||||
password:
|
||||
secretRef: airflow-secrets
|
||||
secretKey: openmetadata-airflow-password
|
||||
|
||||
# Local image configuration
|
||||
image:
|
||||
repository: openmetadata/server
|
||||
tag: "20260107"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Service configuration for local access
|
||||
service:
|
||||
type: LoadBalancer # For minikube tunnel access
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# More relaxed health checks for local testing
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
startupProbe:
|
||||
failureThreshold: 10
|
||||
@@ -0,0 +1,112 @@
|
||||
# Test values for K8s native pipeline client validation
|
||||
# Only contains essential overrides for testing
|
||||
|
||||
openmetadata:
|
||||
config:
|
||||
# Enable debug logging for testing
|
||||
logLevel: DEBUG
|
||||
|
||||
# Database configuration (using local PostgreSQL via host machine)
|
||||
database:
|
||||
host: host.docker.internal
|
||||
port: 5432
|
||||
driverClass: org.postgresql.Driver
|
||||
dbScheme: postgresql
|
||||
auth:
|
||||
password:
|
||||
secretRef: postgres-secrets
|
||||
secretKey: openmetadata-postgres-password
|
||||
|
||||
# Search configuration (using local OpenSearch via host machine)
|
||||
elasticsearch:
|
||||
host: host.docker.internal
|
||||
port: 9200
|
||||
searchType: opensearch
|
||||
scheme: http
|
||||
|
||||
# K8s Pipeline Client Configuration - Updated structure with common configs
|
||||
pipelineServiceClientConfig:
|
||||
type: "k8s" # Use Kubernetes native instead of Airflow
|
||||
# Common configuration for all pipeline service clients
|
||||
# Override metadataApiEndpoint for cross-namespace access (default uses service name only)
|
||||
metadataApiEndpoint: "http://openmetadata.openmetadata-ingestion.svc.cluster.local:8585/api"
|
||||
k8s:
|
||||
# K8s-specific configuration
|
||||
# Use release namespace (empty = defaults to release namespace)
|
||||
namespace: ""
|
||||
serviceAccountName: "openmetadata-ingestion"
|
||||
ttlSecondsAfterFinished: 10
|
||||
|
||||
# Local ingestion image
|
||||
ingestionImage: "openmetadata/collate-base:1.12.0-20260402"
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "2Gi"
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "512Mi"
|
||||
|
||||
# Test tolerations for ingestion pods
|
||||
tolerations:
|
||||
- key: "dedicated"
|
||||
operator: "Equal"
|
||||
value: "ingestion"
|
||||
effect: "NoSchedule"
|
||||
|
||||
# Enable OMJob operator for guaranteed exit handler execution
|
||||
useOMJobOperator: true
|
||||
|
||||
# Local image configuration
|
||||
image:
|
||||
repository: openmetadata/server
|
||||
tag: "1.12.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Service configuration for local access
|
||||
service:
|
||||
type: LoadBalancer # For minikube tunnel access
|
||||
|
||||
# OMJob Operator configuration
|
||||
omjobOperator:
|
||||
enabled: true
|
||||
# Use locally built fixed image
|
||||
image:
|
||||
repository: docker.getcollate.io/openmetadata/omjob-operator
|
||||
tag: "1.12.0-SNAPSHOT"
|
||||
pullPolicy: IfNotPresent
|
||||
# Adjusted resources for Java application (was OOMKilled at 128Mi)
|
||||
resources:
|
||||
requests:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "1Gi"
|
||||
# Debug logging for testing
|
||||
env:
|
||||
logLevel: "DEBUG"
|
||||
watchNamespaces: ""
|
||||
# Enable health checks now that operator implements proper endpoints
|
||||
healthCheck:
|
||||
enabled: true
|
||||
|
||||
# Reduced resources for local testing
|
||||
resources:
|
||||
limits:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# More relaxed health checks for local testing
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 120
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 120
|
||||
startupProbe:
|
||||
failureThreshold: 10
|
||||
@@ -0,0 +1,12 @@
|
||||
FROM node:20-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json ./
|
||||
RUN npm install --production
|
||||
|
||||
COPY server.js ./
|
||||
|
||||
EXPOSE 9090
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "mock-oidc-provider",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "Mock OIDC identity provider for OpenMetadata E2E SSO testing",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"oidc-provider": "^8.4.6",
|
||||
"jose": "^5.2.4",
|
||||
"express": "^4.18.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* Copyright 2025 Collate.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const { Provider } = require('oidc-provider');
|
||||
const express = require('express');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '9090', 10);
|
||||
const ISSUER = process.env.ISSUER || `http://localhost:${PORT}`;
|
||||
|
||||
// Mutable test state — controlled via /test/* endpoints
|
||||
const DEFAULT_LOGIN_ACCOUNT = 'admin';
|
||||
|
||||
const testState = {
|
||||
accessTokenTTL: 3600,
|
||||
idTokenTTL: 3600,
|
||||
forceInteractionRequired: false,
|
||||
refreshTokenEnabled: true,
|
||||
tokenEndpointError: null, // { errorCode: 'invalid_grant', httpStatus: 400 }
|
||||
// Account auto-approved on the next interactive login. Lets a test exercise a
|
||||
// specific identity (e.g. one whose sub differs from the email local-part).
|
||||
defaultLoginAccount: DEFAULT_LOGIN_ACCOUNT,
|
||||
};
|
||||
|
||||
const resetTestState = () => {
|
||||
testState.accessTokenTTL = 3600;
|
||||
testState.idTokenTTL = 3600;
|
||||
testState.forceInteractionRequired = false;
|
||||
testState.refreshTokenEnabled = true;
|
||||
testState.tokenEndpointError = null;
|
||||
testState.defaultLoginAccount = DEFAULT_LOGIN_ACCOUNT;
|
||||
};
|
||||
|
||||
// Request metrics — reset via /test/metrics/reset
|
||||
const metrics = { tokenRequests: 0, authRequests: 0, refreshAttempts: 0 };
|
||||
|
||||
// Pre-seeded test accounts — auto-approved, no interactive login
|
||||
const TEST_ACCOUNTS = new Map([
|
||||
[
|
||||
'admin',
|
||||
{
|
||||
email: 'admin@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Test Admin',
|
||||
sub: 'admin',
|
||||
preferred_username: 'admin',
|
||||
},
|
||||
],
|
||||
[
|
||||
'user1',
|
||||
{
|
||||
email: 'user1@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Test User 1',
|
||||
sub: 'user1',
|
||||
preferred_username: 'user1',
|
||||
},
|
||||
],
|
||||
// Identity whose sub ('claim-user') deliberately differs from the email
|
||||
// local-part ('claim.user.mapped'). Used to verify OIDC self-signup persists
|
||||
// the mapped email claim instead of deriving <sub>@<domain>. See issue #29189.
|
||||
[
|
||||
'claim-user',
|
||||
{
|
||||
email: 'claim.user.mapped@open-metadata.org',
|
||||
email_verified: true,
|
||||
name: 'Claim Mapped User',
|
||||
sub: 'claim-user',
|
||||
preferred_username: 'claim-user',
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const findAccount = (_ctx, id) => {
|
||||
const account = TEST_ACCOUNTS.get(id);
|
||||
if (!account) return undefined;
|
||||
return {
|
||||
accountId: id,
|
||||
async claims(_use, _scope) {
|
||||
return { sub: id, ...account };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const clients = [
|
||||
{
|
||||
client_id: 'openmetadata-test',
|
||||
client_secret: 'openmetadata-test-secret',
|
||||
grant_types: ['authorization_code', 'refresh_token'],
|
||||
redirect_uris: [
|
||||
'http://localhost:8585/callback',
|
||||
'http://localhost:3000/callback',
|
||||
'http://localhost:8585/silent-callback',
|
||||
],
|
||||
post_logout_redirect_uris: [
|
||||
'http://localhost:8585',
|
||||
'http://localhost:3000',
|
||||
],
|
||||
response_types: ['code'],
|
||||
token_endpoint_auth_method: 'client_secret_post',
|
||||
scope: 'openid email profile offline_access',
|
||||
},
|
||||
{
|
||||
client_id: 'openmetadata-test-public',
|
||||
grant_types: ['authorization_code', 'refresh_token'],
|
||||
redirect_uris: [
|
||||
'http://localhost:8585/callback',
|
||||
'http://localhost:3000/callback',
|
||||
'http://localhost:8585/silent-callback',
|
||||
],
|
||||
post_logout_redirect_uris: [
|
||||
'http://localhost:8585',
|
||||
'http://localhost:3000',
|
||||
],
|
||||
response_types: ['code'],
|
||||
token_endpoint_auth_method: 'none',
|
||||
scope: 'openid email profile offline_access',
|
||||
},
|
||||
];
|
||||
|
||||
const providerConfig = {
|
||||
clients,
|
||||
findAccount,
|
||||
claims: {
|
||||
openid: ['sub'],
|
||||
email: ['email', 'email_verified'],
|
||||
profile: ['name', 'preferred_username'],
|
||||
},
|
||||
scopes: ['openid', 'email', 'profile', 'offline_access'],
|
||||
features: {
|
||||
devInteractions: { enabled: false },
|
||||
rpInitiatedLogout: { enabled: true },
|
||||
},
|
||||
pkce: {
|
||||
required: () => false,
|
||||
},
|
||||
ttl: {
|
||||
AccessToken: (_ctx, _token, _client) => testState.accessTokenTTL,
|
||||
IdToken: (_ctx, _token, _client) => testState.idTokenTTL,
|
||||
RefreshToken: (_ctx, _token, _client) =>
|
||||
testState.refreshTokenEnabled ? 86400 : 0,
|
||||
AuthorizationCode: 600,
|
||||
Session: 86400,
|
||||
Grant: 86400,
|
||||
Interaction: 600,
|
||||
},
|
||||
cookies: {
|
||||
keys: [crypto.randomBytes(32).toString('hex')],
|
||||
},
|
||||
jwks: {
|
||||
keys: [
|
||||
{
|
||||
kty: 'RSA',
|
||||
kid: 'mock-oidc-key-1',
|
||||
use: 'sig',
|
||||
alg: 'RS256',
|
||||
// Generated at startup; see init() below
|
||||
},
|
||||
],
|
||||
},
|
||||
// Auto-approve all interactions (skip login/consent screens)
|
||||
interactions: {
|
||||
url(_ctx, _interaction) {
|
||||
return `/interaction/${_interaction.uid}`;
|
||||
},
|
||||
},
|
||||
renderError: async (ctx, out, _error) => {
|
||||
ctx.type = 'html';
|
||||
ctx.body = `<html><body><pre>${JSON.stringify(out, null, 2)}</pre></body></html>`;
|
||||
},
|
||||
};
|
||||
|
||||
async function init() {
|
||||
const { generateKeyPair, exportJWK } = await import('jose');
|
||||
const { privateKey, publicKey } = await generateKeyPair('RS256');
|
||||
const privateJwk = await exportJWK(privateKey);
|
||||
const publicJwk = await exportJWK(publicKey);
|
||||
|
||||
privateJwk.kid = 'mock-oidc-key-1';
|
||||
privateJwk.use = 'sig';
|
||||
privateJwk.alg = 'RS256';
|
||||
|
||||
publicJwk.kid = 'mock-oidc-key-1';
|
||||
publicJwk.use = 'sig';
|
||||
publicJwk.alg = 'RS256';
|
||||
|
||||
providerConfig.jwks = { keys: [privateJwk] };
|
||||
|
||||
const provider = new Provider(ISSUER, providerConfig);
|
||||
|
||||
// Auto-approve interaction: when the OIDC library redirects to /interaction/:uid,
|
||||
// we automatically log in as 'admin' and grant consent without any user input.
|
||||
provider.use(async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
if (
|
||||
ctx.path.startsWith('/interaction/') &&
|
||||
ctx.method === 'GET' &&
|
||||
!ctx.path.endsWith('/abort')
|
||||
) {
|
||||
const uid = ctx.path.split('/interaction/')[1];
|
||||
if (!uid || uid.includes('/')) return;
|
||||
|
||||
try {
|
||||
const interactionDetails = await provider.interactionDetails(
|
||||
ctx.req,
|
||||
ctx.res
|
||||
);
|
||||
const { prompt } = interactionDetails;
|
||||
|
||||
if (testState.forceInteractionRequired) {
|
||||
testState.forceInteractionRequired = false;
|
||||
ctx.status = 403;
|
||||
ctx.body = JSON.stringify({
|
||||
error: 'interaction_required',
|
||||
error_description:
|
||||
'Forced interaction_required for testing silent renewal failure',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (prompt.name === 'login') {
|
||||
const loginUser =
|
||||
ctx.query.login_hint ||
|
||||
interactionDetails.params.login_hint ||
|
||||
testState.defaultLoginAccount ||
|
||||
'admin';
|
||||
const result = {
|
||||
login: { accountId: loginUser },
|
||||
};
|
||||
await provider.interactionFinished(ctx.req, ctx.res, result, {
|
||||
mergeWithLastSubmission: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (prompt.name === 'consent') {
|
||||
const grant = new provider.Grant({
|
||||
accountId: interactionDetails.session.accountId,
|
||||
clientId: interactionDetails.params.client_id,
|
||||
});
|
||||
|
||||
const requestedScopes = interactionDetails.params.scope;
|
||||
if (requestedScopes) {
|
||||
grant.addOIDCScope(requestedScopes);
|
||||
}
|
||||
grant.addOIDCScope('openid email profile offline_access');
|
||||
|
||||
const grantId = await grant.save();
|
||||
|
||||
const result = { consent: { grantId } };
|
||||
await provider.interactionFinished(ctx.req, ctx.res, result, {
|
||||
mergeWithLastSubmission: true,
|
||||
});
|
||||
}
|
||||
} catch (_err) {
|
||||
// Interaction may have already been handled
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
|
||||
// Server-facing base URL for endpoints that the OM server calls from
|
||||
// inside Docker (token exchange, JWKS, userinfo). Defaults to ISSUER
|
||||
// so that outside-Docker usage (tests hitting localhost) works unchanged.
|
||||
const INTERNAL_BASE =
|
||||
process.env.INTERNAL_BASE_URL || ISSUER;
|
||||
|
||||
// Custom discovery endpoint returning hybrid URLs:
|
||||
// - Browser-facing (authorization, end_session): ISSUER (localhost:9090)
|
||||
// - Server-facing (token, jwks, userinfo): INTERNAL_BASE (mock-oidc-provider:9090 in Docker)
|
||||
// This is mounted before oidc-provider so it takes precedence.
|
||||
app.get('/.well-known/openid-configuration', (_req, res) => {
|
||||
res.json({
|
||||
issuer: ISSUER,
|
||||
authorization_endpoint: `${ISSUER}/auth`,
|
||||
token_endpoint: `${INTERNAL_BASE}/token`,
|
||||
jwks_uri: `${INTERNAL_BASE}/jwks`,
|
||||
userinfo_endpoint: `${INTERNAL_BASE}/me`,
|
||||
end_session_endpoint: `${ISSUER}/session/end`,
|
||||
pushed_authorization_request_endpoint: `${INTERNAL_BASE}/request`,
|
||||
claims_parameter_supported: false,
|
||||
claims_supported: [
|
||||
'sub',
|
||||
'email',
|
||||
'email_verified',
|
||||
'name',
|
||||
'preferred_username',
|
||||
'sid',
|
||||
'auth_time',
|
||||
'iss',
|
||||
],
|
||||
code_challenge_methods_supported: ['S256'],
|
||||
grant_types_supported: [
|
||||
'implicit',
|
||||
'authorization_code',
|
||||
'refresh_token',
|
||||
],
|
||||
response_modes_supported: ['form_post', 'fragment', 'query'],
|
||||
response_types_supported: ['code id_token', 'code', 'id_token', 'none'],
|
||||
scopes_supported: ['openid', 'email', 'profile', 'offline_access'],
|
||||
subject_types_supported: ['public'],
|
||||
id_token_signing_alg_values_supported: ['RS256'],
|
||||
token_endpoint_auth_methods_supported: [
|
||||
'client_secret_basic',
|
||||
'client_secret_jwt',
|
||||
'client_secret_post',
|
||||
'private_key_jwt',
|
||||
'none',
|
||||
],
|
||||
token_endpoint_auth_signing_alg_values_supported: [
|
||||
'HS256',
|
||||
'RS256',
|
||||
'PS256',
|
||||
'ES256',
|
||||
'EdDSA',
|
||||
],
|
||||
authorization_response_iss_parameter_supported: true,
|
||||
request_parameter_supported: false,
|
||||
request_uri_parameter_supported: false,
|
||||
claim_types_supported: ['normal'],
|
||||
});
|
||||
});
|
||||
|
||||
// Test control endpoints
|
||||
app.post('/test/configure', (req, res) => {
|
||||
const {
|
||||
accessTokenTTL,
|
||||
idTokenTTL,
|
||||
refreshTokenEnabled,
|
||||
forceInteractionRequired,
|
||||
tokenEndpointError,
|
||||
defaultLoginAccount,
|
||||
} = req.body;
|
||||
if (accessTokenTTL !== undefined)
|
||||
testState.accessTokenTTL = accessTokenTTL;
|
||||
if (idTokenTTL !== undefined) testState.idTokenTTL = idTokenTTL;
|
||||
if (refreshTokenEnabled !== undefined)
|
||||
testState.refreshTokenEnabled = refreshTokenEnabled;
|
||||
if (forceInteractionRequired !== undefined)
|
||||
testState.forceInteractionRequired = forceInteractionRequired;
|
||||
if (tokenEndpointError !== undefined)
|
||||
testState.tokenEndpointError = tokenEndpointError;
|
||||
if (defaultLoginAccount !== undefined)
|
||||
testState.defaultLoginAccount = defaultLoginAccount;
|
||||
|
||||
res.json({ ok: true, state: { ...testState } });
|
||||
});
|
||||
|
||||
app.post('/test/force-interaction-required', (_req, res) => {
|
||||
testState.forceInteractionRequired = true;
|
||||
res.json({ ok: true, forceInteractionRequired: true });
|
||||
});
|
||||
|
||||
app.post('/test/reset', (_req, res) => {
|
||||
resetTestState();
|
||||
res.json({ ok: true, state: { ...testState } });
|
||||
});
|
||||
|
||||
app.get('/test/state', (_req, res) => {
|
||||
res.json({ ...testState });
|
||||
});
|
||||
|
||||
// Metrics endpoints
|
||||
app.get('/test/metrics', (_req, res) => {
|
||||
res.json({ ...metrics });
|
||||
});
|
||||
|
||||
app.post('/test/metrics/reset', (_req, res) => {
|
||||
metrics.tokenRequests = 0;
|
||||
metrics.authRequests = 0;
|
||||
metrics.refreshAttempts = 0;
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// Token endpoint interceptor — must be BEFORE oidc-provider mount.
|
||||
// Tracks metrics and optionally injects errors for testing.
|
||||
app.post('/token', (req, res, next) => {
|
||||
metrics.tokenRequests++;
|
||||
|
||||
const grantType = req.body?.grant_type;
|
||||
if (grantType === 'refresh_token') {
|
||||
metrics.refreshAttempts++;
|
||||
}
|
||||
|
||||
if (testState.tokenEndpointError) {
|
||||
const { errorCode, httpStatus } = testState.tokenEndpointError;
|
||||
testState.tokenEndpointError = null; // one-shot
|
||||
return res.status(httpStatus).json({ error: errorCode });
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
// Auth endpoint interceptor — tracks authorization request metrics.
|
||||
app.get('/auth', (req, res, next) => {
|
||||
metrics.authRequests++;
|
||||
next();
|
||||
});
|
||||
|
||||
// Health check
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({ status: 'ok' });
|
||||
});
|
||||
|
||||
// Mount the OIDC provider
|
||||
app.use(provider.callback());
|
||||
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(
|
||||
`Mock OIDC Provider listening on ${ISSUER}`
|
||||
);
|
||||
console.log(
|
||||
`Discovery: ${ISSUER}/.well-known/openid-configuration`
|
||||
);
|
||||
console.log(`Test control: POST ${ISSUER}/test/configure`);
|
||||
});
|
||||
}
|
||||
|
||||
init().catch((err) => {
|
||||
console.error('Failed to start mock OIDC provider:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
ingestion:
|
||||
container_name: openmetadata_ingestion
|
||||
image: docker.getcollate.io/openmetadata/ingestion:2.0.0-SNAPSHOT
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
|
||||
|
||||
networks:
|
||||
app_net:
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.240.0/24"
|
||||
@@ -0,0 +1,12 @@
|
||||
# Airflow Environment Variables for ingestion service
|
||||
AIRFLOW__API__AUTH_BACKENDS="airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR="LocalExecutor"
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS="/opt/airflow/dag_generated_configs"
|
||||
# MySQL Environment Variables for ingestion service
|
||||
AIRFLOW_DB_HOST="mysql"
|
||||
AIRFLOW_DB_PORT="3306"
|
||||
AIRFLOW_DB="airflow_db"
|
||||
AIRFLOW_DB_SCHEME="mysql+pymysql"
|
||||
AIRFLOW_DB_USER="airflow_user"
|
||||
AIRFLOW_DB_PASSWORD="airflow_pass"
|
||||
AIRFLOW_DB_PROPERTIES=""
|
||||
@@ -0,0 +1,13 @@
|
||||
# Airflow Environment Variables for ingestion service
|
||||
AIRFLOW__API__AUTH_BACKENDS="airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR="LocalExecutor"
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS="/opt/airflow/dag_generated_configs"
|
||||
|
||||
# PostgreSQL Environment Variables for ingestion service
|
||||
AIRFLOW_DB_HOST="postgresql"
|
||||
AIRFLOW_DB_PORT="5432"
|
||||
AIRFLOW_DB="airflow_db"
|
||||
DB_USER="airflow_user"
|
||||
DB_SCHEME="postgresql+psycopg2"
|
||||
DB_PASSWORD="airflow_pass"
|
||||
AIRFLOW_DB_PROPERTIES=""
|
||||
@@ -0,0 +1,465 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
services:
|
||||
execute-migrate-all:
|
||||
container_name: execute_migrate_all
|
||||
command: "./bootstrap/openmetadata-ops.sh migrate"
|
||||
image: docker.getcollate.io/openmetadata/server:2.0.0-SNAPSHOT
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
networks:
|
||||
- app_net
|
||||
|
||||
openmetadata-server:
|
||||
container_name: openmetadata_server
|
||||
restart: always
|
||||
image: docker.getcollate.io/openmetadata/server:2.0.0-SNAPSHOT
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
|
||||
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
networks:
|
||||
app_net:
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.240.0/24"
|
||||
@@ -0,0 +1,170 @@
|
||||
OPENMETADATA_CLUSTER_NAME="openmetadata"
|
||||
SERVER_PORT="8585"
|
||||
SERVER_ADMIN_PORT="8586"
|
||||
LOG_LEVEL="INFO"
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM = 1200
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME="org.openmetadata.service.security.DefaultAuthorizer"
|
||||
AUTHORIZER_REQUEST_FILTER="org.openmetadata.service.security.JwtFilter"
|
||||
AUTHORIZER_ADMIN_PRINCIPALS=[admin]
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN=["all"]
|
||||
AUTHORIZER_INGESTION_PRINCIPALS=[ingestion-bot]
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN="open-metadata.org"
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN="false"
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET="false"
|
||||
AUTHENTICATION_PROVIDER="basic"
|
||||
AUTHENTICATION_RESPONSE_TYPE="id_token"
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=""
|
||||
AUTHENTICATION_PUBLIC_KEYS=[http://localhost:8585/api/v1/system/config/jwks]
|
||||
AUTHENTICATION_AUTHORITY="https://accounts.google.com"
|
||||
AUTHENTICATION_CLIENT_ID=""
|
||||
AUTHENTICATION_CALLBACK_URL=""
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS=[email,preferred_username,sub]
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP="true"
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE="false"
|
||||
# SAML_IDP_ENTITY_ID=""
|
||||
# SAML_IDP_SSO_LOGIN_URL=""
|
||||
# SAML_IDP_CERTIFICATE=""
|
||||
# SAML_AUTHORITY_URL="http://localhost:8585/api/v1/saml/login"
|
||||
# SAML_IDP_NAME_ID="urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"
|
||||
# SAML_SP_ENTITY_ID="http://localhost:8585/api/v1/saml/metadata"
|
||||
# SAML_SP_ACS="http://localhost:8585/api/v1/saml/acs"
|
||||
# SAML_SP_CERTIFICATE=""
|
||||
# SAML_SP_CALLBACK="http://localhost:8585/saml/callback"
|
||||
# SAML_STRICT_MODE="false"
|
||||
# SAML_SP_TOKEN_VALIDITY="3600"
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID="false"
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST="false"
|
||||
# SAML_SIGNED_SP_METADATA="false"
|
||||
# SAML_WANT_MESSAGE_SIGNED="false"
|
||||
# SAML_WANT_ASSERTION_SIGNED="false"
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED="false"
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED="false"
|
||||
# SAML_KEYSTORE_FILE_PATH=""
|
||||
# SAML_KEYSTORE_ALIAS=""
|
||||
# SAML_KEYSTORE_PASSWORD=""
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST=""
|
||||
# AUTHENTICATION_LDAP_PORT=""
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN=""
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD=""
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN=""
|
||||
# AUTHENTICATION_USER_MAIL_ATTR=""
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE="3"
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED=""
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE="TrustAll"
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH=""
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD=""
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT=""
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS=""
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES="[]"
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST=""
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES="true"
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH="./conf/public_key.der"
|
||||
RSA_PRIVATE_KEY_FILE_PATH="./conf/private_key.der"
|
||||
JWT_ISSUER="open-metadata.org"
|
||||
JWT_KEY_ID="Gb389a-9f76-gdjs-a92j-0242bk94356"
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT="http://ingestion:8080"
|
||||
SERVER_HOST_API_URL="http://openmetadata-server:8585/api"
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL="no-ssl"
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH=""
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS="com.mysql.cj.jdbc.Driver"
|
||||
DB_SCHEME="mysql"
|
||||
DB_USE_SSL="false"
|
||||
DB_USER="openmetadata_user"
|
||||
DB_USER_PASSWORD="openmetadata_password"
|
||||
DB_HOST="mysql"
|
||||
DB_PORT="3306"
|
||||
OM_DATABASE="openmetadata_db"
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST= "elasticsearch"
|
||||
ELASTICSEARCH_PORT="9200"
|
||||
ELASTICSEARCH_SCHEME="http"
|
||||
ELASTICSEARCH_USER=""
|
||||
ELASTICSEARCH_PASSWORD=""
|
||||
SEARCH_TYPE="elasticsearch"
|
||||
ELASTICSEARCH_TRUST_STORE_PATH=""
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD=""
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS="5"
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS="60"
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS="600"
|
||||
ELASTICSEARCH_BATCH_SIZE="10"
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE="10485760"
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG="EN"
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION=""
|
||||
AWS_ACCESS_KEY_ID=""
|
||||
AWS_SECRET_ACCESS_KEY=""
|
||||
AWS_SESSION_TOKEN=""
|
||||
SEARCH_AWS_SERVICE_NAME="es"
|
||||
# Event Monitoring configurations
|
||||
EVENT_MONITOR="prometheus"
|
||||
EVENT_MONITOR_BATCH_SIZE="10"
|
||||
EVENT_MONITOR_PATH_PATTERN=["/api/v1/tables/*", "/api/v1/health-check"]
|
||||
EVENT_MONITOR_LATENCY="[]"
|
||||
#PipelineServiceClient Configuration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME="org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED="false"
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP=""
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER="noop"
|
||||
#Airflow Parameters
|
||||
AIRFLOW_USERNAME="admin"
|
||||
AIRFLOW_PASSWORD="admin"
|
||||
AIRFLOW_TIMEOUT="10"
|
||||
AIRFLOW_TRUST_STORE_PATH=""
|
||||
AIRFLOW_TRUST_STORE_PASSWORD=""
|
||||
FERNET_KEY="jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA="
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER="noop"
|
||||
OM_SM_REGION=""
|
||||
OM_SM_ACCESS_KEY_ID=""
|
||||
OM_SM_ACCESS_KEY=""
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY="OpenMetadata"
|
||||
OM_SUPPORT_URL="https://slack.open-metadata.org"
|
||||
AUTHORIZER_ENABLE_SMTP="false"
|
||||
OPENMETADATA_SERVER_URL="http://localhost:8585"
|
||||
OPENMETADATA_SMTP_SENDER_MAIL=""
|
||||
SMTP_SERVER_ENDPOINT=""
|
||||
SMTP_SERVER_PORT=""
|
||||
SMTP_SERVER_USERNAME=""
|
||||
SMTP_SERVER_PWD=""
|
||||
SMTP_SERVER_STRATEGY="SMTP_TLS"
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES="[]"
|
||||
OM_EXTENSIONS="[]"
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS="-Xmx2G -Xms256M -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError"
|
||||
# Application Config
|
||||
CUSTOM_LOGO_URL_PATH=""
|
||||
CUSTOM_MONOGRAM_URL_PATH=""
|
||||
OM_MAX_FAILED_LOGIN_ATTEMPTS=3
|
||||
OM_LOGIN_ACCESS_BLOCK_TIME=600
|
||||
OM_JWT_EXPIRY_TIME=3600
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API="false"
|
||||
#WebConfiguration
|
||||
WEB_CONF_URI_PATH="/api"
|
||||
WEB_CONF_HSTS_ENABLED=false
|
||||
WEB_CONF_HSTS_MAX_AGE="365 days"
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS="true"
|
||||
WEB_CONF_HSTS_PRELOAD="true"
|
||||
WEB_CONF_FRAME_OPTION_ENABLED=false
|
||||
WEB_CONF_FRAME_OPTION="SAMEORIGIN"
|
||||
WEB_CONF_FRAME_ORIGIN=""
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED=false
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED=false
|
||||
WEB_CONF_XSS_PROTECTION_ON=true
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK=true
|
||||
WEB_CONF_XSS_CSP_ENABLED=false
|
||||
WEB_CONF_XSS_CSP_POLICY="default-src 'self'"
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY=""
|
||||
@@ -0,0 +1,170 @@
|
||||
OPENMETADATA_CLUSTER_NAME="openmetadata"
|
||||
SERVER_PORT="8585"
|
||||
SERVER_ADMIN_PORT="8586"
|
||||
LOG_LEVEL="INFO"
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM = 1200
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME="org.openmetadata.service.security.DefaultAuthorizer"
|
||||
AUTHORIZER_REQUEST_FILTER="org.openmetadata.service.security.JwtFilter"
|
||||
AUTHORIZER_ADMIN_PRINCIPALS=[admin]
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN=["all"]
|
||||
AUTHORIZER_INGESTION_PRINCIPALS=[ingestion-bot]
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN="open-metadata.org"
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN="false"
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET="false"
|
||||
AUTHENTICATION_PROVIDER="basic"
|
||||
AUTHENTICATION_RESPONSE_TYPE:"id_token"
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME=""
|
||||
AUTHENTICATION_PUBLIC_KEYS=[http://localhost:8585/api/v1/system/config/jwks]
|
||||
AUTHENTICATION_AUTHORITY="https://accounts.google.com"
|
||||
AUTHENTICATION_CLIENT_ID=""
|
||||
AUTHENTICATION_CALLBACK_URL=""
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS=[email,preferred_username,sub]
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP="true"
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE="false"
|
||||
# SAML_IDP_ENTITY_ID=""
|
||||
# SAML_IDP_SSO_LOGIN_URL=""
|
||||
# SAML_IDP_CERTIFICATE=""
|
||||
# SAML_AUTHORITY_URL="http://localhost:8585/api/v1/saml/login"
|
||||
# SAML_IDP_NAME_ID="urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"
|
||||
# SAML_SP_ENTITY_ID="http://localhost:8585/api/v1/saml/metadata"
|
||||
# SAML_SP_ACS="http://localhost:8585/api/v1/saml/acs"
|
||||
# SAML_SP_CERTIFICATE=""
|
||||
# SAML_SP_CALLBACK="http://localhost:8585/saml/callback"
|
||||
# SAML_STRICT_MODE="false"
|
||||
# SAML_SP_TOKEN_VALIDITY="3600"
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID="false"
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST="false"
|
||||
# SAML_SIGNED_SP_METADATA="false"
|
||||
# SAML_WANT_MESSAGE_SIGNED="false"
|
||||
# SAML_WANT_ASSERTION_SIGNED="false"
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED="false"
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED="false"
|
||||
# SAML_KEYSTORE_FILE_PATH=""
|
||||
# SAML_KEYSTORE_ALIAS=""
|
||||
# SAML_KEYSTORE_PASSWORD=""
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST=""
|
||||
# AUTHENTICATION_LDAP_PORT=""
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN=""
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD=""
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN=""
|
||||
# AUTHENTICATION_USER_MAIL_ATTR=""
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE="3"
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED=""
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE="TrustAll"
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH=""
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD=""
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT=""
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS=""
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES="[]"
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST=""
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES="true"
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH="./conf/public_key.der"
|
||||
RSA_PRIVATE_KEY_FILE_PATH="./conf/private_key.der"
|
||||
JWT_ISSUER="open-metadata.org"
|
||||
JWT_KEY_ID="Gb389a-9f76-gdjs-a92j-0242bk94356"
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT="http://ingestion:8080"
|
||||
SERVER_HOST_API_URL="http://openmetadata-server:8585/api"
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL="no-ssl"
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH=""
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS="org.postgresql.Driver"
|
||||
DB_SCHEME="postgresql"
|
||||
DB_USE_SSL="false"
|
||||
DB_USER="openmetadata_user"
|
||||
DB_USER_PASSWORD="openmetadata_password"
|
||||
DB_HOST="postgresql"
|
||||
DB_PORT="5432"
|
||||
OM_DATABASE="openmetadata_db"
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST= "elasticsearch"
|
||||
ELASTICSEARCH_PORT="9200"
|
||||
ELASTICSEARCH_SCHEME="http"
|
||||
ELASTICSEARCH_USER=""
|
||||
ELASTICSEARCH_PASSWORD=""
|
||||
SEARCH_TYPE="elasticsearch"
|
||||
ELASTICSEARCH_TRUST_STORE_PATH=""
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD=""
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS="5"
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS="60"
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS="600"
|
||||
ELASTICSEARCH_BATCH_SIZE="10"
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE="10485760"
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG="EN"
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION=""
|
||||
AWS_ACCESS_KEY_ID=""
|
||||
AWS_SECRET_ACCESS_KEY=""
|
||||
AWS_SESSION_TOKEN=""
|
||||
SEARCH_AWS_SERVICE_NAME="es"
|
||||
# Event Monitoring configurations
|
||||
EVENT_MONITOR="prometheus"
|
||||
EVENT_MONITOR_BATCH_SIZE="10"
|
||||
EVENT_MONITOR_PATH_PATTERN=["/api/v1/tables/*", "/api/v1/health-check"]
|
||||
EVENT_MONITOR_LATENCY="[]"
|
||||
#PipelineServiceClient Configuration
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME="org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED="false"
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP=""
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER="noop"
|
||||
#Airflow Parameters
|
||||
AIRFLOW_USERNAME="admin"
|
||||
AIRFLOW_PASSWORD="admin"
|
||||
AIRFLOW_TIMEOUT="10"
|
||||
AIRFLOW_TRUST_STORE_PATH=""
|
||||
AIRFLOW_TRUST_STORE_PASSWORD=""
|
||||
FERNET_KEY="jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA="
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER="noop"
|
||||
OM_SM_REGION=""
|
||||
OM_SM_ACCESS_KEY_ID=""
|
||||
OM_SM_ACCESS_KEY=""
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY="OpenMetadata"
|
||||
OM_SUPPORT_URL="https://slack.open-metadata.org"
|
||||
AUTHORIZER_ENABLE_SMTP="false"
|
||||
OPENMETADATA_SERVER_URL="http://localhost:8585"
|
||||
OPENMETADATA_SMTP_SENDER_MAIL=""
|
||||
SMTP_SERVER_ENDPOINT=""
|
||||
SMTP_SERVER_PORT=""
|
||||
SMTP_SERVER_USERNAME=""
|
||||
SMTP_SERVER_PWD=""
|
||||
SMTP_SERVER_STRATEGY="SMTP_TLS"
|
||||
#extensionConfiguration
|
||||
OM_RESOURCE_PACKAGES="[]"
|
||||
OM_EXTENSIONS="[]"
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS="-Xmx2G -Xms512M -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError"
|
||||
# Application Config
|
||||
CUSTOM_LOGO_URL_PATH=""
|
||||
CUSTOM_MONOGRAM_URL_PATH=""
|
||||
OM_MAX_FAILED_LOGIN_ATTEMPTS=3
|
||||
OM_LOGIN_ACCESS_BLOCK_TIME=600
|
||||
OM_JWT_EXPIRY_TIME=3600
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API="false"
|
||||
#WebConfiguration
|
||||
WEB_CONF_URI_PATH="/api"
|
||||
WEB_CONF_HSTS_ENABLED=false
|
||||
WEB_CONF_HSTS_MAX_AGE="365 days"
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS="true"
|
||||
WEB_CONF_HSTS_PRELOAD="true"
|
||||
WEB_CONF_FRAME_OPTION_ENABLED=false
|
||||
WEB_CONF_FRAME_OPTION="SAMEORIGIN"
|
||||
WEB_CONF_FRAME_ORIGIN=""
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED=false
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED=false
|
||||
WEB_CONF_XSS_PROTECTION_ON=true
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK=true
|
||||
WEB_CONF_XSS_CSP_ENABLED=false
|
||||
WEB_CONF_XSS_CSP_POLICY="default-src 'self'"
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY=""
|
||||
@@ -0,0 +1,14 @@
|
||||
# Docker volumes location for RDF services
|
||||
# Default: Uses docker-volumes directory in current path
|
||||
# You can override this to use an external directory like:
|
||||
# DOCKER_VOLUMES_PATH=/home/user/docker-volumes
|
||||
DOCKER_VOLUMES_PATH=./docker-volumes
|
||||
|
||||
# Fuseki settings
|
||||
FUSEKI_ADMIN_PASSWORD=admin
|
||||
FUSEKI_MEMORY_MAX=4g
|
||||
FUSEKI_MEMORY_MIN=2g
|
||||
|
||||
# OpenSearch settings
|
||||
OPENSEARCH_MEMORY_MAX=4g
|
||||
OPENSEARCH_MEMORY_MIN=2g
|
||||
@@ -0,0 +1,48 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build stage
|
||||
FROM alpine:3 AS build
|
||||
ARG RI_VERSION="2.0.0-SNAPSHOT"
|
||||
ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz"
|
||||
|
||||
RUN mkdir -p /opt/openmetadata && \
|
||||
wget ${RELEASE_URL} && \
|
||||
tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
|
||||
rm openmetadata-*.tar.gz
|
||||
|
||||
# Final stage
|
||||
FROM alpine:3
|
||||
ARG RI_VERSION="2.0.0-SNAPSHOT"
|
||||
ARG BUILD_DATE
|
||||
ARG COMMIT_ID
|
||||
LABEL maintainer="OpenMetadata"
|
||||
LABEL org.open-metadata.image.authors="support@openmetadata.org"
|
||||
LABEL org.open-metadata.vendor="OpenMetadata"
|
||||
LABEL org.open-metadata.release-version="$RI_VERSION"
|
||||
LABEL org.open-metadata.description="OpenMetadata is an open source platform for metadata management and discovery."
|
||||
LABEL org.open-metadata.url="https://open-metadata.org/"
|
||||
LABEL org.open-metadata.vcs-url="https://github.com/open-metadata/OpenMetadata"
|
||||
LABEL org.open-metadata.build-date=$BUILD_DATE
|
||||
LABEL org.open-metadata.commit-id=$COMMIT_ID
|
||||
|
||||
EXPOSE 8585 8586
|
||||
RUN adduser -D openmetadata && apk update && \
|
||||
apk upgrade && \
|
||||
apk add --update --no-cache bash openjdk21-jre
|
||||
|
||||
COPY --chown=openmetadata:openmetadata --from=build /opt/openmetadata /opt/openmetadata
|
||||
COPY --chmod=755 docker/openmetadata-start.sh ./
|
||||
USER openmetadata
|
||||
|
||||
WORKDIR /opt/openmetadata
|
||||
ENTRYPOINT [ "/bin/bash" ]
|
||||
CMD ["/openmetadata-start.sh"]
|
||||
@@ -0,0 +1,38 @@
|
||||
# Use eclipse-temurin which supports ARM64
|
||||
FROM eclipse-temurin:17-jre
|
||||
|
||||
# Install minimal packages
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends wget && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Fuseki version and paths
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
ENV FUSEKI_BASE=/fuseki
|
||||
|
||||
# Create fuseki user and directories
|
||||
RUN addgroup -g 1000 -S fuseki && \
|
||||
adduser -u 1000 -S fuseki -G fuseki && \
|
||||
mkdir -p ${FUSEKI_HOME} && \
|
||||
chown -R fuseki:fuseki ${FUSEKI_HOME}
|
||||
|
||||
# Switch to fuseki user
|
||||
USER fuseki
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# Download and install Fuseki
|
||||
RUN wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz --strip-components=1 && \
|
||||
rm apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
mkdir -p ${FUSEKI_HOME}/run ${FUSEKI_HOME}/databases
|
||||
|
||||
# JVM options
|
||||
ENV JVM_ARGS="-Xmx4g -Xms2g"
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3030
|
||||
|
||||
# Start Fuseki with openmetadata dataset
|
||||
CMD ["sh", "-c", "mkdir -p ${FUSEKI_HOME}/databases/openmetadata && exec ${FUSEKI_HOME}/fuseki-server --update --loc=${FUSEKI_HOME}/databases/openmetadata /openmetadata"]
|
||||
@@ -0,0 +1,61 @@
|
||||
# Multi-architecture Dockerfile for Apache Jena Fuseki
|
||||
FROM eclipse-temurin:17-jre-jammy
|
||||
|
||||
# Fix for GPG signature issues and install dependencies
|
||||
RUN apt-get update || true && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
wget \
|
||||
curl \
|
||||
ca-certificates \
|
||||
|| (rm -rf /var/lib/apt/lists/* && \
|
||||
apt-get clean && \
|
||||
apt-get update --fix-missing && \
|
||||
apt-get install -y --no-install-recommends wget curl ca-certificates) && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Fuseki version
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
|
||||
# Download and install Fuseki
|
||||
RUN mkdir -p ${FUSEKI_HOME} && \
|
||||
cd /tmp && \
|
||||
wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
mv apache-jena-fuseki-${FUSEKI_VERSION}/* ${FUSEKI_HOME}/ && \
|
||||
rm -rf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz apache-jena-fuseki-${FUSEKI_VERSION}
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p ${FUSEKI_HOME}/run && \
|
||||
mkdir -p ${FUSEKI_HOME}/databases
|
||||
|
||||
# Set working directory
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# Expose Fuseki port
|
||||
EXPOSE 3030
|
||||
|
||||
# Set JVM options
|
||||
ENV JVM_ARGS="-Xmx4g -Xms2g"
|
||||
|
||||
# Create entrypoint script
|
||||
RUN echo '#!/bin/bash\n\
|
||||
if [ ! -z "$ADMIN_PASSWORD" ]; then\n\
|
||||
echo "Setting admin password..."\n\
|
||||
${FUSEKI_HOME}/fuseki-server --passwd --update /fuseki/run/shiro.ini <<EOF\n\
|
||||
admin=$ADMIN_PASSWORD\n\
|
||||
EOF\n\
|
||||
fi\n\
|
||||
\n\
|
||||
# Create openmetadata database if it doesn'\''t exist\n\
|
||||
mkdir -p ${FUSEKI_HOME}/databases/openmetadata\n\
|
||||
\n\
|
||||
# Start Fuseki\n\
|
||||
exec ${FUSEKI_HOME}/fuseki-server --update --loc=${FUSEKI_HOME}/databases/openmetadata /openmetadata\n\
|
||||
' > /entrypoint.sh && chmod +x /entrypoint.sh
|
||||
|
||||
# Volume for persistent data
|
||||
VOLUME ["${FUSEKI_HOME}/databases", "${FUSEKI_HOME}/run"]
|
||||
|
||||
# Entrypoint
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,36 @@
|
||||
# Multi-architecture Fuseki build
|
||||
# eclipse-temurin replaces the deprecated openjdk Docker Hub images.
|
||||
FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre-jammy
|
||||
|
||||
# Install required packages
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
wget \
|
||||
ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Fuseki version and paths
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
ENV FUSEKI_BASE=/fuseki
|
||||
|
||||
# Download and install Fuseki
|
||||
WORKDIR /tmp
|
||||
RUN wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
mv apache-jena-fuseki-${FUSEKI_VERSION}/* ${FUSEKI_HOME}/ && \
|
||||
rm -rf /tmp/*
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p ${FUSEKI_HOME}/run ${FUSEKI_HOME}/databases
|
||||
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# JVM options
|
||||
ENV JVM_ARGS="-Xmx4g -Xms2g"
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3030
|
||||
|
||||
# Start Fuseki with openmetadata dataset
|
||||
CMD ["sh", "-c", "mkdir -p ${FUSEKI_HOME}/databases/openmetadata && exec ${FUSEKI_HOME}/fuseki-server --update --loc=${FUSEKI_HOME}/databases/openmetadata /openmetadata"]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Simple multi-platform Fuseki using official image
|
||||
FROM --platform=$BUILDPLATFORM alpine:latest AS downloader
|
||||
|
||||
RUN apk add --no-cache wget tar
|
||||
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
WORKDIR /tmp
|
||||
|
||||
RUN wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz && \
|
||||
mv apache-jena-fuseki-${FUSEKI_VERSION} /fuseki-dist
|
||||
|
||||
# Runtime: eclipse-temurin replaces the deprecated openjdk Docker Hub images.
|
||||
FROM eclipse-temurin:17-jre-jammy
|
||||
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
ENV FUSEKI_BASE=/fuseki
|
||||
|
||||
# Copy Fuseki from builder
|
||||
COPY --from=downloader /fuseki-dist ${FUSEKI_HOME}
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p ${FUSEKI_HOME}/run ${FUSEKI_HOME}/databases
|
||||
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3030
|
||||
|
||||
# Default JVM options
|
||||
ENV JVM_ARGS="-Xmx4g -Xms2g"
|
||||
|
||||
# Simple entrypoint
|
||||
CMD ["sh", "-c", "mkdir -p ${FUSEKI_HOME}/databases/openmetadata && exec ${FUSEKI_HOME}/fuseki-server --update --loc=${FUSEKI_HOME}/databases/openmetadata /openmetadata"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Simple Fuseki build that works on ARM64
|
||||
# eclipse-temurin replaces the deprecated openjdk Docker Hub images.
|
||||
FROM eclipse-temurin:17-jre-jammy
|
||||
|
||||
# Set Fuseki version and paths
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
ENV FUSEKI_BASE=/fuseki
|
||||
|
||||
# Create directories
|
||||
RUN mkdir -p ${FUSEKI_HOME}
|
||||
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# Download Fuseki using ADD (doesn't require wget)
|
||||
ADD https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz /tmp/fuseki.tar.gz
|
||||
|
||||
# Extract Fuseki
|
||||
RUN tar -xzf /tmp/fuseki.tar.gz --strip-components=1 -C ${FUSEKI_HOME} && \
|
||||
rm /tmp/fuseki.tar.gz && \
|
||||
mkdir -p ${FUSEKI_HOME}/run ${FUSEKI_HOME}/databases
|
||||
|
||||
# JVM options
|
||||
ENV JVM_ARGS="-Xmx4g -Xms2g"
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3030
|
||||
|
||||
# Start Fuseki
|
||||
CMD ["sh", "-c", "mkdir -p ${FUSEKI_HOME}/databases/openmetadata && exec ${FUSEKI_HOME}/fuseki-server --update --loc=${FUSEKI_HOME}/databases/openmetadata /openmetadata"]
|
||||
@@ -0,0 +1,51 @@
|
||||
version: "3.9"
|
||||
|
||||
# ARM64-native Apache Jena Fuseki for RDF/Knowledge Graph storage
|
||||
services:
|
||||
fuseki:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.fuseki-simple
|
||||
platforms:
|
||||
- linux/arm64
|
||||
- linux/amd64
|
||||
container_name: fuseki-standalone
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Admin credentials
|
||||
- ADMIN_PASSWORD=admin
|
||||
# JVM memory settings - adjust based on your system
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
# Fuseki configuration
|
||||
- FUSEKI_HOME=/fuseki
|
||||
volumes:
|
||||
# Mount directory for persistent storage (configurable via .env)
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki/databases:/fuseki/databases
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki/run:/fuseki/run
|
||||
networks:
|
||||
- fuseki-net
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3030/$/ping"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
networks:
|
||||
fuseki-net:
|
||||
driver: bridge
|
||||
|
||||
# Usage:
|
||||
# 1. Create the volume directory: mkdir -p fuseki-volume/run
|
||||
# 2. Build and start: docker-compose -f docker-compose-fuseki-arm64.yml up -d --build
|
||||
# 3. Access Fuseki UI: http://localhost:3030
|
||||
# 4. Login: admin/admin
|
||||
# 5. SPARQL endpoint: http://localhost:3030/openmetadata/sparql
|
||||
@@ -0,0 +1,39 @@
|
||||
version: "3.9"
|
||||
|
||||
# Official Apache Jena Fuseki with multi-platform support
|
||||
services:
|
||||
fuseki:
|
||||
# Build multi-architecture image locally using Alpine
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.fuseki-working
|
||||
container_name: fuseki-standalone
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Admin credentials
|
||||
- ADMIN_PASSWORD=admin
|
||||
# JVM memory settings
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
volumes:
|
||||
# Mount directory for persistent storage
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki:/fuseki
|
||||
networks:
|
||||
- fuseki-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:3030/$/ping"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
networks:
|
||||
fuseki-net:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,47 @@
|
||||
# Fuseki using Rosetta 2 emulation on Apple Silicon
|
||||
services:
|
||||
fuseki:
|
||||
# Force AMD64 platform with Rosetta 2 emulation
|
||||
platform: linux/amd64
|
||||
# Build from the in-repo Dockerfile (Fuseki 5.6.0). See
|
||||
# docker-compose-fuseki.yml for the full rationale.
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: fuseki-standalone
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Local-dev default — production deployments MUST override via
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=-Xmx8g -Xms4g
|
||||
volumes:
|
||||
# Host bind path renamed from `.../fuseki` (used by the old stain image
|
||||
# layout) to `.../fuseki-tdb2-data` so an existing host directory with
|
||||
# the previous layout isn't silently mounted at /fuseki-data — Fuseki
|
||||
# would see an empty TDB2 store and the old data would appear lost.
|
||||
# Operators upgrading can either delete the new dir to start fresh or
|
||||
# migrate old data manually.
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki-tdb2-data:/fuseki-data
|
||||
networks:
|
||||
- fuseki-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 10G
|
||||
reservations:
|
||||
memory: 8G
|
||||
|
||||
networks:
|
||||
fuseki-net:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,49 @@
|
||||
# Standalone Apache Jena Fuseki for RDF/Knowledge Graph storage
|
||||
services:
|
||||
fuseki:
|
||||
# Build from the in-repo Dockerfile (Fuseki 5.6.0). See
|
||||
# ../development/docker-compose-fuseki.yml for the full rationale.
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
container_name: fuseki-standalone
|
||||
hostname: fuseki
|
||||
ports:
|
||||
- "3030:3030"
|
||||
environment:
|
||||
# Local-dev default — production deployments MUST override via
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
volumes:
|
||||
# See docker-compose-fuseki-rosetta.yml — host bind path renamed so
|
||||
# existing directories with the old stain layout aren't silently
|
||||
# mounted at the new /fuseki-data path.
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/fuseki-tdb2-data:/fuseki-data
|
||||
networks:
|
||||
- fuseki-net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
networks:
|
||||
fuseki-net:
|
||||
driver: bridge
|
||||
|
||||
# Usage:
|
||||
# 1. Create the volume directory: mkdir -p fuseki-volume
|
||||
# 2. Start: docker-compose -f docker-compose-fuseki-standalone.yml up -d
|
||||
# 3. Access Fuseki UI: http://localhost:3030
|
||||
# 4. Login: admin/admin
|
||||
# 5. SPARQL endpoint: http://localhost:3030/openmetadata/sparql
|
||||
@@ -0,0 +1,84 @@
|
||||
# Standalone OpenSearch 3.0 with security disabled for local testing
|
||||
services:
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.4.0
|
||||
platform: linux/arm64 # Native ARM64 support
|
||||
container_name: opensearch-standalone
|
||||
hostname: opensearch
|
||||
environment:
|
||||
# Cluster settings
|
||||
- cluster.name=opensearch-cluster
|
||||
- node.name=opensearch-node1
|
||||
- discovery.type=single-node
|
||||
- bootstrap.memory_lock=true
|
||||
|
||||
# Disable security for local testing
|
||||
- DISABLE_SECURITY_PLUGIN=true
|
||||
- DISABLE_INSTALL_DEMO_CONFIG=true
|
||||
|
||||
# JVM memory settings - adjust based on your system
|
||||
- OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx4g
|
||||
|
||||
# Performance settings
|
||||
- indices.memory.index_buffer_size=20%
|
||||
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||
|
||||
# Compatibility mode for Elasticsearch clients
|
||||
- compatibility.override_main_response_version=true
|
||||
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
volumes:
|
||||
# Mount directory for persistent storage (configurable via .env)
|
||||
- ${DOCKER_VOLUMES_PATH:-./docker-volumes}/opensearch:/usr/share/opensearch/data
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9600:9600" # Performance analyzer
|
||||
networks:
|
||||
- opensearch-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
# Optional: OpenSearch Dashboards for visualization
|
||||
# Commented out to save space - uncomment if needed
|
||||
# opensearch-dashboards:
|
||||
# image: opensearchproject/opensearch-dashboards:2.19.0
|
||||
# platform: linux/arm64 # Native ARM64 support
|
||||
# container_name: opensearch-dashboards
|
||||
# ports:
|
||||
# - "5601:5601"
|
||||
# environment:
|
||||
# - OPENSEARCH_HOSTS=["http://opensearch:9200"]
|
||||
# - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
|
||||
# networks:
|
||||
# - opensearch-net
|
||||
# depends_on:
|
||||
# opensearch:
|
||||
# condition: service_healthy
|
||||
|
||||
networks:
|
||||
opensearch-net:
|
||||
driver: bridge
|
||||
|
||||
# Usage:
|
||||
# 1. Set vm.max_map_count: sudo sysctl -w vm.max_map_count=262144
|
||||
# 2. Create the volume directory: mkdir -p search-volume
|
||||
# 3. Start: docker-compose -f docker-compose-opensearch-standalone.yml up -d
|
||||
# 4. Access OpenSearch: http://localhost:9200
|
||||
# 5. Access OpenSearch Dashboards: http://localhost:5601
|
||||
# 6. No authentication required (security disabled)
|
||||
@@ -0,0 +1,559 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
postgresql:
|
||||
container_name: openmetadata_postgresql
|
||||
image: docker.getcollate.io/openmetadata/postgresql:2.0.0-SNAPSHOT
|
||||
restart: always
|
||||
command: "--work_mem=10MB"
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: password
|
||||
expose:
|
||||
- 5432
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ./docker-volume/db-data-postgres:/var/lib/postgresql/data
|
||||
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: psql -U postgres -tAc 'select 1' -d openmetadata_db
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
elasticsearch:
|
||||
container_name: openmetadata_elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- app_net
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
container_name: execute_migrate_all
|
||||
image: docker.getcollate.io/openmetadata/server:2.0.0-SNAPSHOT
|
||||
command: "./bootstrap/openmetadata-ops.sh migrate"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- app_net
|
||||
|
||||
openmetadata-server:
|
||||
container_name: openmetadata_server
|
||||
restart: always
|
||||
image: docker.getcollate.io/openmetadata/server:2.0.0-SNAPSHOT
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
#Database configuration for postgresql
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-postgresql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-postgresql}
|
||||
DB_PORT: ${DB_PORT:-5432}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
container_name: openmetadata_ingestion
|
||||
image: docker.getcollate.io/openmetadata/ingestion:2.0.0-SNAPSHOT
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-5432}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-postgresql+psycopg2}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
|
||||
networks:
|
||||
app_net:
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.240.0/24"
|
||||
@@ -0,0 +1,509 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# OpenMetadata Quickstart with RDF/Knowledge Graph support
|
||||
# Usage: docker-compose -f docker-compose-rdf.yml up
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
# See ../development/docker-compose-fuseki.yml — renamed from `fuseki-data`
|
||||
# because the new Dockerfile uses a different on-disk layout and reusing
|
||||
# the old volume name silently mounts stale state.
|
||||
fuseki-tdb2-data:
|
||||
|
||||
services:
|
||||
mysql:
|
||||
container_name: openmetadata_mysql
|
||||
image: docker.getcollate.io/openmetadata/db:1.8.0-SNAPSHOT
|
||||
command: "--sort_buffer_size=10M"
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
elasticsearch:
|
||||
container_name: openmetadata_elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms2g -Xmx4g
|
||||
- xpack.security.enabled=false
|
||||
- indices.memory.index_buffer_size=20%
|
||||
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||
networks:
|
||||
- app_net
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
# Apache Jena Fuseki for RDF/Knowledge Graph storage. Built from the
|
||||
# in-repo Dockerfile (Fuseki 5.6.0) — see ../development/docker-compose-fuseki.yml
|
||||
# for why we don't use the unmaintained stain/jena-fuseki image.
|
||||
fuseki:
|
||||
container_name: openmetadata_fuseki
|
||||
build:
|
||||
context: ../rdf-store
|
||||
dockerfile: Dockerfile
|
||||
image: openmetadata-fuseki:5.6.0
|
||||
restart: always
|
||||
environment:
|
||||
# Local-dev defaults — production deployments MUST override via the
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD env vars before
|
||||
# bringing this stack up. The entrypoint envsubsts these into shiro.ini.
|
||||
- FUSEKI_ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
|
||||
- FUSEKI_OPENMETADATA_PASSWORD=${FUSEKI_OPENMETADATA_PASSWORD:-openmetadata-secret}
|
||||
- JVM_ARGS=-Xmx4g -Xms2g
|
||||
ports:
|
||||
- "3030:3030"
|
||||
volumes:
|
||||
- fuseki-tdb2-data:/fuseki-data
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 40s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
|
||||
execute-migrate-all:
|
||||
container_name: execute_migrate_all
|
||||
image: docker.getcollate.io/openmetadata/server:1.8.0-SNAPSHOT
|
||||
command: "./bootstrap/openmetadata-ops.sh migrate"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-RS256}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-""}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-""}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
|
||||
# Elasticsearch configuration
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-20}
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:-elasticsearch}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
ELASTICSEARCH_CA_CERT_PATH: ${ELASTICSEARCH_CA_CERT_PATH:-""}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
|
||||
ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES: ${ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES:-10485760}
|
||||
ELASTICSEARCH_INDEX_FACTORY: ${ELASTICSEARCH_INDEX_FACTORY:-"org.openmetadata.service.search.elasticsearch.ElasticSearchIndexFactory"}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx2G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- app_net
|
||||
|
||||
openmetadata-server:
|
||||
container_name: openmetadata_server
|
||||
restart: always
|
||||
image: docker.getcollate.io/openmetadata/server:1.8.0-SNAPSHOT
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-RS256}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-""}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-""}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_USE_SSL: ${DB_USE_SSL:-false}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
|
||||
# Elasticsearch configuration
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-20}
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:-elasticsearch}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
ELASTICSEARCH_CA_CERT_PATH: ${ELASTICSEARCH_CA_CERT_PATH:-""}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
|
||||
ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES: ${ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES:-10485760}
|
||||
ELASTICSEARCH_INDEX_FACTORY: ${ELASTICSEARCH_INDEX_FACTORY:-"org.openmetadata.service.search.elasticsearch.ElasticSearchIndexFactory"}
|
||||
|
||||
# RDF Configuration
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
|
||||
RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
|
||||
RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
|
||||
RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
|
||||
RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
|
||||
RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}
|
||||
|
||||
# Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx2G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
container_name: openmetadata_ingestion
|
||||
build:
|
||||
context: ../
|
||||
dockerfile: docker/development/Dockerfile
|
||||
image: docker.getcollate.io/openmetadata/ingestion:1.8.0-SNAPSHOT
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_healthy
|
||||
fuseki:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
|
||||
# RDF Configuration (for any RDF-aware ingestion pipelines)
|
||||
RDF_ENABLED: ${RDF_ENABLED:-true}
|
||||
RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
|
||||
RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
|
||||
RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
|
||||
RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
|
||||
networks:
|
||||
app_net:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,44 @@
|
||||
version: "3.9"
|
||||
|
||||
# Override file to increase disk storage for persistent volumes
|
||||
# Place this file in the same directory as docker-compose-rdf.yml
|
||||
|
||||
volumes:
|
||||
# Increase Elasticsearch data volume
|
||||
es-data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: ./docker-volume/elasticsearch-data
|
||||
|
||||
# Fuseki data volume. Renamed from `fuseki-data` (and host path changed
|
||||
# from `./docker-volume/fuseki-data` to `./docker-volume/fuseki-tdb2-data`)
|
||||
# to match docker-compose-rdf.yml — see ../development/docker-compose-fuseki.yml
|
||||
# for the migration rationale (new on-disk layout vs the old stain image).
|
||||
fuseki-tdb2-data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: none
|
||||
o: bind
|
||||
device: ./docker-volume/fuseki-tdb2-data
|
||||
|
||||
services:
|
||||
# Additional Elasticsearch optimizations
|
||||
elasticsearch:
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
sysctls:
|
||||
- vm.max_map_count=262144
|
||||
|
||||
# Additional Fuseki optimizations
|
||||
fuseki:
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
@@ -0,0 +1,559 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3.9"
|
||||
volumes:
|
||||
ingestion-volume-dag-airflow:
|
||||
ingestion-volume-dags:
|
||||
ingestion-volume-tmp:
|
||||
es-data:
|
||||
services:
|
||||
mysql:
|
||||
container_name: openmetadata_mysql
|
||||
image: ${OPENMETADATA_DB_IMAGE:-docker.getcollate.io/openmetadata/db:1.12.0-SNAPSHOT}
|
||||
command: "--sort_buffer_size=10M"
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: password
|
||||
expose:
|
||||
- 3306
|
||||
ports:
|
||||
- "3306:3306"
|
||||
volumes:
|
||||
- ./docker-volume/db-data:/var/lib/mysql
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
|
||||
elasticsearch:
|
||||
container_name: openmetadata_elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
|
||||
- xpack.security.enabled=false
|
||||
networks:
|
||||
- app_net
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9300:9300"
|
||||
healthcheck:
|
||||
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
volumes:
|
||||
- es-data:/usr/share/elasticsearch/data
|
||||
|
||||
execute-migrate-all:
|
||||
container_name: execute_migrate_all
|
||||
image: ${OPENMETADATA_SERVER_IMAGE:-docker.getcollate.io/openmetadata/server:1.12.0-SNAPSHOT}
|
||||
command: "./bootstrap/openmetadata-ops.sh migrate"
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# Migration
|
||||
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
# AWS:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
# Azure:
|
||||
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
|
||||
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
|
||||
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
|
||||
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Referrer-Policy
|
||||
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
|
||||
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
|
||||
#Permission-Policy
|
||||
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
|
||||
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- app_net
|
||||
|
||||
openmetadata-server:
|
||||
container_name: openmetadata_server
|
||||
restart: always
|
||||
image: ${OPENMETADATA_SERVER_IMAGE:-docker.getcollate.io/openmetadata/server:1.12.0-SNAPSHOT}
|
||||
environment:
|
||||
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
|
||||
SERVER_PORT: ${SERVER_PORT:-8585}
|
||||
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
|
||||
# OpenMetadata Server Authentication Configuration
|
||||
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
|
||||
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
|
||||
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
|
||||
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
|
||||
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
|
||||
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
|
||||
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
|
||||
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
|
||||
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
|
||||
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
|
||||
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
|
||||
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
|
||||
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
|
||||
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
|
||||
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
|
||||
#For OIDC Authentication, when client is confidential
|
||||
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
|
||||
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
|
||||
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
|
||||
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
|
||||
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
|
||||
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
|
||||
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
|
||||
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
|
||||
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
|
||||
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
|
||||
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
|
||||
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
|
||||
OIDC_TENANT: ${OIDC_TENANT:-""}
|
||||
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
|
||||
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
|
||||
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
|
||||
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
|
||||
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_SESSION_EXPIRY: ${AUTHENTICATION_SESSION_EXPIRY:-"604800"}
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}
|
||||
# For SAML Authentication
|
||||
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
|
||||
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
|
||||
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
|
||||
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
|
||||
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
|
||||
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
|
||||
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
|
||||
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
|
||||
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
|
||||
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
|
||||
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
|
||||
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
|
||||
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
|
||||
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
|
||||
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
|
||||
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
|
||||
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
|
||||
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
|
||||
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
|
||||
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
|
||||
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
|
||||
# For LDAP Authentication
|
||||
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
|
||||
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
|
||||
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
|
||||
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
|
||||
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
|
||||
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
|
||||
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
|
||||
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
|
||||
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
|
||||
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
|
||||
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
|
||||
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
|
||||
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
|
||||
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
|
||||
|
||||
# JWT Configuration
|
||||
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
|
||||
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
|
||||
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
|
||||
# OpenMetadata Server Pipeline Service Client Configuration
|
||||
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
|
||||
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
|
||||
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
|
||||
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
|
||||
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
|
||||
# Database configuration for MySQL
|
||||
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
|
||||
DB_SCHEME: ${DB_SCHEME:-mysql}
|
||||
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
|
||||
DB_USER: ${DB_USER:-openmetadata_user}
|
||||
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
|
||||
DB_HOST: ${DB_HOST:-mysql}
|
||||
DB_PORT: ${DB_PORT:-3306}
|
||||
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
|
||||
# ElasticSearch Configurations
|
||||
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
|
||||
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
|
||||
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
|
||||
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
|
||||
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
|
||||
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
|
||||
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
|
||||
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
|
||||
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
|
||||
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
|
||||
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
|
||||
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
|
||||
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
|
||||
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
|
||||
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
|
||||
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
|
||||
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
|
||||
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
|
||||
|
||||
#eventMonitoringConfiguration
|
||||
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
|
||||
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
|
||||
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
|
||||
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
|
||||
|
||||
#pipelineServiceClientConfiguration
|
||||
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
|
||||
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
|
||||
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
|
||||
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
|
||||
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
|
||||
#airflow parameters
|
||||
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
|
||||
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
|
||||
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
|
||||
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
|
||||
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
|
||||
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
|
||||
|
||||
#secretsManagerConfiguration
|
||||
SECRET_MANAGER: ${SECRET_MANAGER:-db}
|
||||
#parameters:
|
||||
OM_SM_REGION: ${OM_SM_REGION:-""}
|
||||
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
|
||||
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
|
||||
|
||||
#email configuration:
|
||||
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
|
||||
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
|
||||
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
|
||||
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
|
||||
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
|
||||
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
|
||||
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
|
||||
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
|
||||
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
|
||||
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
|
||||
|
||||
# Heap OPTS Configurations
|
||||
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
|
||||
# Mask passwords values in UI
|
||||
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
|
||||
|
||||
#OpenMetadata Web Configuration
|
||||
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
|
||||
#HSTS
|
||||
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
|
||||
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
|
||||
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
|
||||
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
|
||||
#Frame Options
|
||||
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
|
||||
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
|
||||
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
|
||||
#Content Type
|
||||
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
|
||||
#XSS-Protection
|
||||
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
|
||||
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
|
||||
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
|
||||
#CSP
|
||||
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
|
||||
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
|
||||
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
|
||||
#Cache
|
||||
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
|
||||
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
|
||||
|
||||
expose:
|
||||
- 8585
|
||||
- 8586
|
||||
ports:
|
||||
- "8585:8585"
|
||||
- "8586:8586"
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
execute-migrate-all:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- app_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
|
||||
|
||||
ingestion:
|
||||
container_name: openmetadata_ingestion
|
||||
image: ${OPENMETADATA_INGESTION_IMAGE:-docker.getcollate.io/openmetadata/ingestion:1.12.0-SNAPSHOT}
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
openmetadata-server:
|
||||
condition: service_started
|
||||
environment:
|
||||
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
|
||||
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
|
||||
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
|
||||
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
|
||||
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
|
||||
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
|
||||
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
|
||||
# extra connection-string properties for the database
|
||||
# EXAMPLE
|
||||
# require SSL (only for Postgres)
|
||||
# properties: "?sslmode=require"
|
||||
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
|
||||
# To test the lineage backend
|
||||
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
|
||||
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
|
||||
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
|
||||
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
|
||||
entrypoint: /bin/bash
|
||||
command:
|
||||
- "/opt/airflow/ingestion_dependency.sh"
|
||||
expose:
|
||||
- 8080
|
||||
ports:
|
||||
- "8080:8080"
|
||||
networks:
|
||||
- app_net
|
||||
volumes:
|
||||
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
|
||||
- ingestion-volume-dags:/opt/airflow/dags
|
||||
- ingestion-volume-tmp:/tmp
|
||||
|
||||
|
||||
networks:
|
||||
app_net:
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: "172.16.240.0/24"
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to start Fuseki and OpenSearch services for OpenMetadata RDF development
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== Starting RDF Services for OpenMetadata ==="
|
||||
|
||||
# Check if running on macOS or Linux
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo "Running on macOS - skipping vm.max_map_count setting"
|
||||
else
|
||||
# Set vm.max_map_count for OpenSearch (Linux only)
|
||||
echo "Setting vm.max_map_count for OpenSearch..."
|
||||
sudo sysctl -w vm.max_map_count=262144 || {
|
||||
echo "Warning: Could not set vm.max_map_count. OpenSearch might have issues."
|
||||
echo "Try running: sudo sysctl -w vm.max_map_count=262144"
|
||||
}
|
||||
fi
|
||||
|
||||
# Load environment variables if .env exists
|
||||
if [ -f .env ]; then
|
||||
echo "Loading environment variables from .env..."
|
||||
export $(grep -v '^#' .env | xargs)
|
||||
fi
|
||||
|
||||
# Use default path if not set
|
||||
VOLUMES_PATH=${DOCKER_VOLUMES_PATH:-./docker-volumes}
|
||||
|
||||
# Create volume directories
|
||||
echo "Creating volume directories in: $VOLUMES_PATH"
|
||||
mkdir -p "$VOLUMES_PATH/fuseki/databases" "$VOLUMES_PATH/fuseki/run" "$VOLUMES_PATH/opensearch"
|
||||
|
||||
# Set permissions
|
||||
echo "Setting permissions..."
|
||||
chmod -R 777 "$VOLUMES_PATH"
|
||||
|
||||
# Start Fuseki
|
||||
echo "Starting Apache Jena Fuseki..."
|
||||
# Use Rosetta 2 emulation on ARM64 because the local Fuseki Dockerfile
|
||||
# bases on a Linux x86_64 image; the Rosetta variant pins platform=linux/amd64.
|
||||
if [[ $(uname -m) == "arm64" ]] || [[ $(uname -m) == "aarch64" ]]; then
|
||||
echo "Detected ARM64 architecture, using Rosetta 2 emulation..."
|
||||
docker compose -f docker-compose-fuseki-rosetta.yml up -d
|
||||
else
|
||||
docker compose -f docker-compose-fuseki-standalone.yml up -d
|
||||
fi
|
||||
|
||||
# Start OpenSearch
|
||||
echo "Starting OpenSearch..."
|
||||
docker compose -f docker-compose-opensearch-standalone.yml up -d
|
||||
|
||||
# Wait for services to be healthy
|
||||
echo "Waiting for services to start..."
|
||||
sleep 10
|
||||
|
||||
# Check Fuseki
|
||||
echo -n "Checking Fuseki status... "
|
||||
if curl -s http://localhost:3030/$/ping > /dev/null; then
|
||||
echo "✓ Fuseki is running"
|
||||
echo " - Fuseki UI: http://localhost:3030"
|
||||
echo " - SPARQL endpoint: http://localhost:3030/openmetadata/sparql"
|
||||
echo " - Login: admin/admin"
|
||||
else
|
||||
echo "✗ Fuseki is not responding"
|
||||
fi
|
||||
|
||||
# Check OpenSearch
|
||||
echo -n "Checking OpenSearch status... "
|
||||
if curl -s http://localhost:9200/_cluster/health > /dev/null; then
|
||||
echo "✓ OpenSearch is running"
|
||||
echo " - OpenSearch API: http://localhost:9200"
|
||||
echo " - OpenSearch Dashboards: http://localhost:5601"
|
||||
echo " - No authentication required (security disabled)"
|
||||
|
||||
# Show cluster health
|
||||
echo ""
|
||||
echo "OpenSearch cluster health:"
|
||||
curl -s http://localhost:9200/_cluster/health?pretty | head -10
|
||||
else
|
||||
echo "✗ OpenSearch is not responding"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Services Started ==="
|
||||
echo ""
|
||||
echo "To stop services:"
|
||||
echo " docker-compose -f docker-compose-fuseki-standalone.yml down"
|
||||
echo " docker-compose -f docker-compose-opensearch-standalone.yml down"
|
||||
echo ""
|
||||
echo "To view logs:"
|
||||
echo " docker logs -f fuseki-standalone"
|
||||
echo " docker logs -f opensearch-standalone"
|
||||
echo ""
|
||||
echo "To clean up data:"
|
||||
echo " rm -rf fuseki-volume/* search-volume/*"
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to stop Fuseki and OpenSearch services
|
||||
|
||||
echo "=== Stopping RDF Services ==="
|
||||
|
||||
# Stop Fuseki
|
||||
echo "Stopping Apache Jena Fuseki..."
|
||||
# Try to stop all possible Fuseki configurations
|
||||
docker compose -f docker-compose-fuseki-rosetta.yml down 2>/dev/null || true
|
||||
docker compose -f docker-compose-fuseki-multiarch.yml down 2>/dev/null || true
|
||||
docker compose -f docker-compose-fuseki-arm64.yml down 2>/dev/null || true
|
||||
docker compose -f docker-compose-fuseki-standalone.yml down 2>/dev/null || true
|
||||
|
||||
# Stop OpenSearch
|
||||
echo "Stopping OpenSearch..."
|
||||
docker compose -f docker-compose-opensearch-standalone.yml down
|
||||
|
||||
# Load environment variables if .env exists
|
||||
if [ -f .env ]; then
|
||||
export $(grep -v '^#' .env | xargs)
|
||||
fi
|
||||
|
||||
# Use default path if not set
|
||||
VOLUMES_PATH=${DOCKER_VOLUMES_PATH:-./docker-volumes}
|
||||
|
||||
echo ""
|
||||
echo "=== Services Stopped ==="
|
||||
echo ""
|
||||
echo "Data is preserved in:"
|
||||
echo " - Fuseki: $VOLUMES_PATH/fuseki"
|
||||
echo " - OpenSearch: $VOLUMES_PATH/opensearch"
|
||||
echo ""
|
||||
echo "To remove all data:"
|
||||
echo " rm -rf $VOLUMES_PATH/*"
|
||||
echo ""
|
||||
echo "To remove Docker volumes:"
|
||||
echo " docker volume prune"
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for RDF services
|
||||
|
||||
echo "=== Testing RDF Services ==="
|
||||
echo ""
|
||||
|
||||
# Test Fuseki
|
||||
echo "Testing Apache Jena Fuseki..."
|
||||
echo -n " Checking health: "
|
||||
if curl -s http://localhost:3030/$/ping > /dev/null 2>&1; then
|
||||
echo "✓ OK"
|
||||
|
||||
echo -n " Checking datasets: "
|
||||
if curl -s http://localhost:3030/$/datasets | grep -q "openmetadata"; then
|
||||
echo "✓ openmetadata dataset found"
|
||||
else
|
||||
echo "✗ openmetadata dataset not found"
|
||||
fi
|
||||
|
||||
echo -n " Testing SPARQL endpoint: "
|
||||
SPARQL_TEST=$(curl -s -X POST http://localhost:3030/openmetadata/sparql \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
-d 'query=SELECT ?s WHERE { ?s ?p ?o } LIMIT 1' 2>&1)
|
||||
if echo "$SPARQL_TEST" | grep -q "results"; then
|
||||
echo "✓ SPARQL endpoint working"
|
||||
else
|
||||
echo "✗ SPARQL endpoint not responding correctly"
|
||||
fi
|
||||
else
|
||||
echo "✗ Fuseki not responding on port 3030"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Test OpenSearch
|
||||
echo "Testing OpenSearch..."
|
||||
echo -n " Checking health: "
|
||||
HEALTH=$(curl -s http://localhost:9200/_cluster/health 2>/dev/null)
|
||||
if echo "$HEALTH" | grep -q "status"; then
|
||||
STATUS=$(echo "$HEALTH" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
echo "✓ OK (status: $STATUS)"
|
||||
|
||||
echo -n " Checking version: "
|
||||
VERSION=$(curl -s http://localhost:9200 | grep -o '"number":"[^"]*"' | cut -d'"' -f4)
|
||||
echo "✓ Version $VERSION"
|
||||
|
||||
echo -n " Checking indices: "
|
||||
INDICES=$(curl -s http://localhost:9200/_cat/indices?v 2>/dev/null | wc -l)
|
||||
echo "✓ $((INDICES-1)) indices"
|
||||
else
|
||||
echo "✗ OpenSearch not responding on port 9200"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Test Complete ==="
|
||||
@@ -0,0 +1,46 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Dockerfile - minimal-ubuntu
|
||||
# usage: docker build -t minimal-ubuntu:0.1 .
|
||||
|
||||
FROM ubuntu:xenial-20210416
|
||||
|
||||
# environment variables
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# update
|
||||
RUN apt update -y && \
|
||||
# editor
|
||||
apt install -y vim nano && \
|
||||
# general
|
||||
apt install -y man sudo sshpass less jq ntp bc && \
|
||||
# network commands
|
||||
apt install -y net-tools iputils-ping dnsutils lsof curl wget telnet
|
||||
|
||||
# python
|
||||
RUN apt install -y python-dev && \
|
||||
curl "https://bootstrap.pypa.io/get-pip.py" -o /tmp/get-pip.py && \
|
||||
python /tmp/get-pip.py
|
||||
|
||||
# java
|
||||
RUN apt install -y openjdk-21-jdk
|
||||
# maven
|
||||
#RUN apt install -y maven=3.3.9-3
|
||||
|
||||
# supervisord
|
||||
RUN pip install supervisor==3.3.3 && \
|
||||
mkdir -p /var/log/supervisord/
|
||||
|
||||
# converts the dash to bash terminal for easy scripting
|
||||
RUN update-alternatives --install /bin/sh sh /bin/bash 100 && \
|
||||
update-alternatives --install /bin/sh sh /bin/dash 200 && \
|
||||
echo 1 | update-alternatives --config sh
|
||||
@@ -0,0 +1,22 @@
|
||||
# Keycloak SAML Fixture
|
||||
|
||||
Local SAML IdP fixture for the Playwright SSO login spec.
|
||||
|
||||
```bash
|
||||
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
It imports one realm for an OpenMetadata server running at `http://localhost:8585`:
|
||||
|
||||
- `om-azure-saml`
|
||||
- User: `azure.saml@openmetadata.local`
|
||||
- Password: `OpenMetadata@123`
|
||||
|
||||
Use the matching Playwright provider type:
|
||||
|
||||
```bash
|
||||
SSO_PROVIDER_TYPE=keycloak-azure-saml \
|
||||
SSO_USERNAME=azure.saml@openmetadata.local \
|
||||
SSO_PASSWORD=OpenMetadata@123 \
|
||||
npx playwright test playwright/e2e/Auth/SSOLogin.spec.ts --project=sso-auth --workers=1
|
||||
```
|
||||
@@ -0,0 +1,23 @@
|
||||
name: openmetadata-keycloak-saml
|
||||
|
||||
services:
|
||||
keycloak:
|
||||
image: ${KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak:26.3.3}
|
||||
container_name: openmetadata-keycloak-saml
|
||||
command: ['start-dev', '--import-realm']
|
||||
environment:
|
||||
KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_BOOTSTRAP_ADMIN_USERNAME:-admin}
|
||||
KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_BOOTSTRAP_ADMIN_PASSWORD:-admin123}
|
||||
KC_HEALTH_ENABLED: 'true'
|
||||
KC_HTTP_ENABLED: 'true'
|
||||
KC_HTTP_PORT: '8080'
|
||||
ports:
|
||||
- '${KEYCLOAK_SAML_PORT:-8080}:8080'
|
||||
volumes:
|
||||
- ./realms:/opt/keycloak/data/import:ro
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'exec 3<>/dev/tcp/localhost/8080']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 18
|
||||
start_period: 20s
|
||||
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"realm": "om-azure-saml",
|
||||
"enabled": true,
|
||||
"displayName": "OpenMetadata Azure SAML",
|
||||
"sslRequired": "none",
|
||||
"registrationAllowed": false,
|
||||
"loginWithEmailAllowed": true,
|
||||
"duplicateEmailsAllowed": false,
|
||||
"resetPasswordAllowed": false,
|
||||
"editUsernameAllowed": false,
|
||||
"clients": [
|
||||
{
|
||||
"clientId": "http://localhost:8585/api/v1/saml/metadata",
|
||||
"name": "OpenMetadata",
|
||||
"enabled": true,
|
||||
"protocol": "saml",
|
||||
"publicClient": true,
|
||||
"frontchannelLogout": true,
|
||||
"redirectUris": ["http://localhost:8585/*"],
|
||||
"baseUrl": "http://localhost:8585",
|
||||
"adminUrl": "http://localhost:8585",
|
||||
"attributes": {
|
||||
"saml.assertion.signature": "true",
|
||||
"saml.authnstatement": "true",
|
||||
"saml.client.signature": "false",
|
||||
"saml.encrypt": "false",
|
||||
"saml.force.name.id.format": "true",
|
||||
"saml.force.post.binding": "true",
|
||||
"saml.multivalued.roles": "false",
|
||||
"saml.server.signature": "true",
|
||||
"saml.signature.algorithm": "RSA_SHA256",
|
||||
"saml_assertion_consumer_url_post": "http://localhost:8585/api/v1/saml/acs",
|
||||
"saml_force_name_id_format": "true",
|
||||
"saml_name_id_format": "email"
|
||||
},
|
||||
"protocolMappers": [
|
||||
{
|
||||
"name": "Email",
|
||||
"protocol": "saml",
|
||||
"protocolMapper": "saml-user-property-mapper",
|
||||
"consentRequired": false,
|
||||
"config": {
|
||||
"attribute.name": "email",
|
||||
"attribute.nameformat": "Basic",
|
||||
"friendly.name": "email",
|
||||
"user.attribute": "email"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Display Name",
|
||||
"protocol": "saml",
|
||||
"protocolMapper": "saml-user-attribute-mapper",
|
||||
"consentRequired": false,
|
||||
"config": {
|
||||
"attribute.name": "http://schemas.microsoft.com/identity/claims/displayname",
|
||||
"attribute.nameformat": "Basic",
|
||||
"friendly.name": "displayname",
|
||||
"user.attribute": "displayName"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Given Name",
|
||||
"protocol": "saml",
|
||||
"protocolMapper": "saml-user-property-mapper",
|
||||
"consentRequired": false,
|
||||
"config": {
|
||||
"attribute.name": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname",
|
||||
"attribute.nameformat": "Basic",
|
||||
"friendly.name": "givenname",
|
||||
"user.attribute": "firstName"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Surname",
|
||||
"protocol": "saml",
|
||||
"protocolMapper": "saml-user-property-mapper",
|
||||
"consentRequired": false,
|
||||
"config": {
|
||||
"attribute.name": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname",
|
||||
"attribute.nameformat": "Basic",
|
||||
"friendly.name": "surname",
|
||||
"user.attribute": "lastName"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"users": [
|
||||
{
|
||||
"username": "azure.saml@openmetadata.local",
|
||||
"email": "azure.saml@openmetadata.local",
|
||||
"firstName": "Azure",
|
||||
"lastName": "SAML",
|
||||
"enabled": true,
|
||||
"emailVerified": true,
|
||||
"requiredActions": [],
|
||||
"attributes": {
|
||||
"displayName": ["Azure SAML User"]
|
||||
},
|
||||
"credentials": [
|
||||
{
|
||||
"type": "password",
|
||||
"value": "OpenMetadata@123",
|
||||
"temporary": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
FROM mysql/mysql-server:latest
|
||||
WORKDIR /docker-entrypoint-initdb.d
|
||||
COPY docker/mysql/mysql-script.sql .
|
||||
RUN chmod -R 775 /docker-entrypoint-initdb.d
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE DATABASE openmetadata_db;
|
||||
CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
||||
CREATE USER 'openmetadata_user'@'%' IDENTIFIED BY 'openmetadata_password';
|
||||
CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass';
|
||||
GRANT ALL PRIVILEGES ON openmetadata_db.* TO 'openmetadata_user'@'%' WITH GRANT OPTION;
|
||||
GRANT PROCESS, USAGE ON *.* TO 'openmetadata_user'@'%';
|
||||
GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%' WITH GRANT OPTION;
|
||||
GRANT PROCESS, USAGE ON *.* TO 'openmetadata_user'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
commit;
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
echo "Initializing OpenMetadata Server...";
|
||||
echo " ||||||| "
|
||||
echo " |||| |||| ____ "
|
||||
echo " |||| |||| / __ \ "
|
||||
echo " |||| |||| | | | | _ __ ___ _ __ "
|
||||
echo " ||||| ||||| | | | || '_ \ / _ \| '_ \ "
|
||||
echo " ||||||||||||| | |__| || |_) || __/| | | | "
|
||||
echo " ||||||||||||| \____/ | .__/ \___||_| |_| "
|
||||
echo " ||| ||||| ||| __ __ | | _ _ _ "
|
||||
echo " ||| ||| ||| | \/ ||_| | | | | | | "
|
||||
echo " ||| | ||| | \ / | ___ | |_ __ _ __| | __ _ | |_ __ _ "
|
||||
echo " ||| ||| | |\/| | / _ \| __|/ _\` | / _\` | / _\` || __|/ _\` | "
|
||||
echo " ||| || || ||| | | | || __/| |_| (_| || (_| || (_| || |_| (_| | "
|
||||
echo " ||| ||||| ||| |_| |_| \___| \__|\__,_| \__,_| \__,_| \__|\__,_| "
|
||||
echo " ||||||||||| "
|
||||
echo " ||||||| "
|
||||
echo "Starting OpenMetadata Server"
|
||||
./bin/openmetadata-server-start.sh conf/openmetadata.yaml
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
conf/openmetadata.yaml
|
||||
@@ -0,0 +1,4 @@
|
||||
FROM postgres:15
|
||||
WORKDIR /docker-entrypoint-initdb.d
|
||||
COPY docker/postgresql/postgres-script.sql .
|
||||
RUN chmod -R 775 /docker-entrypoint-initdb.d
|
||||
@@ -0,0 +1,8 @@
|
||||
CREATE DATABASE openmetadata_db;
|
||||
CREATE DATABASE airflow_db;
|
||||
CREATE USER openmetadata_user WITH PASSWORD 'openmetadata_password';
|
||||
CREATE USER airflow_user WITH PASSWORD 'airflow_pass';
|
||||
ALTER DATABASE openmetadata_db OWNER TO openmetadata_user;
|
||||
ALTER DATABASE airflow_db OWNER TO airflow_user;
|
||||
ALTER USER airflow_user SET search_path = public;
|
||||
commit;
|
||||
@@ -0,0 +1,59 @@
|
||||
# Apache Jena Fuseki Docker Image for OpenMetadata RDF Store
|
||||
# eclipse-temurin replaces the deprecated `openjdk` Docker Hub images
|
||||
# (`openjdk:17-jdk-slim` was removed from the registry — CI builds against it
|
||||
# fail with "manifest unknown"). JRE is enough since this image only runs the
|
||||
# Fuseki shell launcher; no compilation happens inside the container.
|
||||
FROM eclipse-temurin:17-jre-jammy
|
||||
|
||||
ENV FUSEKI_VERSION=5.6.0
|
||||
ENV FUSEKI_HOME=/fuseki
|
||||
# FUSEKI_BASE must point at the directory containing shiro.ini so Fuseki picks
|
||||
# up our auth config on boot. Without this, Fuseki falls back to its built-in
|
||||
# default base (typically the working directory) and the bundled shiro.ini is
|
||||
# never loaded — leaving the admin endpoints (incl. /$/compact and
|
||||
# /$/datasets) reachable without authentication. The Dockerfile copies
|
||||
# config.ttl + shiro.ini into /fuseki below, so we point FUSEKI_BASE there.
|
||||
ENV FUSEKI_BASE=/fuseki
|
||||
|
||||
# gettext-base provides `envsubst`, used by the entrypoint to inject
|
||||
# FUSEKI_ADMIN_PASSWORD / FUSEKI_OPENMETADATA_PASSWORD into shiro.ini at
|
||||
# container start. Without this, operators could not override the default
|
||||
# Fuseki credentials via environment variables.
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
gettext-base \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Download and install Fuseki
|
||||
RUN wget -q https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz \
|
||||
&& tar -xzf apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz \
|
||||
&& mv apache-jena-fuseki-${FUSEKI_VERSION} ${FUSEKI_HOME} \
|
||||
&& rm apache-jena-fuseki-${FUSEKI_VERSION}.tar.gz
|
||||
|
||||
WORKDIR ${FUSEKI_HOME}
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /fuseki-data
|
||||
|
||||
# Custom configuration. shiro.ini ships as a TEMPLATE because Apache Shiro's
|
||||
# INI realm does not interpolate ${VAR} placeholders natively — we have to
|
||||
# render it at container start with the actual passwords. The entrypoint
|
||||
# does that via envsubst.
|
||||
COPY config.ttl /fuseki/config.ttl
|
||||
COPY shiro.ini.template /fuseki/shiro.ini.template
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
# Expose Fuseki port
|
||||
EXPOSE 3030
|
||||
|
||||
# Volume for persistent data
|
||||
VOLUME ["/fuseki-data"]
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \
|
||||
CMD wget -q --spider http://localhost:3030/$/ping || exit 1
|
||||
|
||||
# Run Fuseki via the entrypoint (which renders shiro.ini then execs fuseki-server)
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["./fuseki-server", "--loc=/fuseki-data", "--update", "/openmetadata"]
|
||||
@@ -0,0 +1,47 @@
|
||||
# Fuseki configuration for OpenMetadata RDF Store
|
||||
|
||||
@prefix : <#> .
|
||||
@prefix fuseki: <http://jena.apache.org/fuseki#> .
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix tdb2: <http://jena.apache.org/2016/tdb#> .
|
||||
@prefix ja: <http://jena.hpl.hp.com/2005/11/Assembler#> .
|
||||
|
||||
[] rdf:type fuseki:Server ;
|
||||
fuseki:services (
|
||||
<#openmetadata>
|
||||
) .
|
||||
|
||||
<#openmetadata> rdf:type fuseki:Service ;
|
||||
fuseki:name "openmetadata" ;
|
||||
fuseki:serviceQuery "sparql" ;
|
||||
fuseki:serviceQuery "query" ;
|
||||
fuseki:serviceUpdate "update" ;
|
||||
fuseki:serviceUpload "upload" ;
|
||||
fuseki:serviceReadWriteGraphStore "data" ;
|
||||
fuseki:dataset <#dataset> .
|
||||
|
||||
<#dataset> rdf:type tdb2:DatasetTDB2 ;
|
||||
tdb2:location "/fuseki-data/openmetadata" ;
|
||||
tdb2:unionDefaultGraph true ;
|
||||
.
|
||||
|
||||
# Enable full-text search
|
||||
<#dataset> ja:textIndex (<#index>) .
|
||||
|
||||
<#index> a text:TextIndexLucene ;
|
||||
text:directory "/fuseki-data/lucene" ;
|
||||
text:entityMap <#entityMap> ;
|
||||
text:storeValues true .
|
||||
|
||||
<#entityMap> a text:EntityMap ;
|
||||
text:defaultField "text" ;
|
||||
text:entityField "uri" ;
|
||||
text:map (
|
||||
[ text:field "text" ;
|
||||
text:predicate rdfs:label ]
|
||||
[ text:field "description" ;
|
||||
text:predicate <http://purl.org/dc/terms/description> ]
|
||||
[ text:field "name" ;
|
||||
text:predicate <https://open-metadata.org/ontology/name> ]
|
||||
) .
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Render shiro.ini from its template, substituting FUSEKI_ADMIN_PASSWORD and
|
||||
# FUSEKI_OPENMETADATA_PASSWORD. Apache Shiro's INI realm does not interpolate
|
||||
# ${VAR} placeholders natively, so we have to expand them before Fuseki reads
|
||||
# the file — otherwise Shiro stores the literal string `${FUSEKI_...}` as the
|
||||
# password and every basic-auth attempt returns 401.
|
||||
#
|
||||
# Defaults: admin / admin and openmetadata / openmetadata-secret. Operators
|
||||
# who want different credentials set the env vars in their compose / k8s
|
||||
# deployment manifest — that override now actually takes effect.
|
||||
#
|
||||
# Operators who need to fully replace shiro.ini (different role layout,
|
||||
# custom realms, …) have two options:
|
||||
#
|
||||
# 1. Bind-mount your file onto /fuseki/shiro.ini AND set
|
||||
# FUSEKI_RENDER_SHIRO=false — the entrypoint then skips the
|
||||
# envsubst render and leaves the mounted file in place.
|
||||
#
|
||||
# 2. Bind-mount onto /fuseki/shiro.ini.template instead of /fuseki/shiro.ini
|
||||
# and the entrypoint will envsubst your template (handy if you want
|
||||
# env-driven password injection in your custom realm too).
|
||||
#
|
||||
# Defaulting FUSEKI_RENDER_SHIRO=true preserves the prior, password-injection
|
||||
# behavior for every dev/quickstart compose deployment that doesn't override
|
||||
# it.
|
||||
|
||||
set -eu
|
||||
|
||||
: "${FUSEKI_ADMIN_PASSWORD:=admin}"
|
||||
: "${FUSEKI_OPENMETADATA_PASSWORD:=openmetadata-secret}"
|
||||
: "${FUSEKI_RENDER_SHIRO:=true}"
|
||||
export FUSEKI_ADMIN_PASSWORD FUSEKI_OPENMETADATA_PASSWORD
|
||||
|
||||
if [ "$FUSEKI_RENDER_SHIRO" = "true" ] && [ -f /fuseki/shiro.ini.template ]; then
|
||||
# Restrict envsubst to the two variables we expect. Without an explicit
|
||||
# list, envsubst would interpret any `${...}` in the template — including
|
||||
# comments — which would silently blank out unrelated placeholders if
|
||||
# they were ever added.
|
||||
envsubst '${FUSEKI_ADMIN_PASSWORD} ${FUSEKI_OPENMETADATA_PASSWORD}' \
|
||||
</fuseki/shiro.ini.template \
|
||||
>/fuseki/shiro.ini
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
@@ -0,0 +1,98 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: openmetadata-fuseki
|
||||
labels:
|
||||
app: openmetadata-fuseki
|
||||
spec:
|
||||
ports:
|
||||
- port: 3030
|
||||
targetPort: 3030
|
||||
name: fuseki
|
||||
selector:
|
||||
app: openmetadata-fuseki
|
||||
type: ClusterIP
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: fuseki-data-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
storageClassName: standard
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: openmetadata-fuseki
|
||||
labels:
|
||||
app: openmetadata-fuseki
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: openmetadata-fuseki
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: openmetadata-fuseki
|
||||
spec:
|
||||
containers:
|
||||
- name: fuseki
|
||||
image: openmetadata/fuseki-rdf:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 3030
|
||||
name: fuseki
|
||||
env:
|
||||
- name: FUSEKI_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: fuseki-secrets
|
||||
key: admin-password
|
||||
- name: FUSEKI_OPENMETADATA_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: fuseki-secrets
|
||||
key: openmetadata-password
|
||||
- name: JAVA_OPTIONS
|
||||
value: "-Xmx2g -Xms1g"
|
||||
volumeMounts:
|
||||
- name: fuseki-data
|
||||
mountPath: /fuseki-data
|
||||
resources:
|
||||
requests:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "3Gi"
|
||||
cpu: "2000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /$/ping
|
||||
port: 3030
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /$/ping
|
||||
port: 3030
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: fuseki-data
|
||||
persistentVolumeClaim:
|
||||
claimName: fuseki-data-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: fuseki-secrets
|
||||
type: Opaque
|
||||
stringData:
|
||||
admin-password: "your-secure-admin-password"
|
||||
openmetadata-password: "your-secure-service-password"
|
||||
@@ -0,0 +1,62 @@
|
||||
# Apache Shiro configuration for Fuseki security — TEMPLATE FILE.
|
||||
#
|
||||
# Do NOT mount or copy this file as-is into /fuseki/shiro.ini. It contains
|
||||
# `${FUSEKI_ADMIN_PASSWORD}` and `${FUSEKI_OPENMETADATA_PASSWORD}` placeholders
|
||||
# that Apache Shiro's INI realm does NOT interpolate — if Fuseki loads the
|
||||
# raw template, the literal string `${FUSEKI_ADMIN_PASSWORD}` becomes the
|
||||
# admin password, which silently lets `${FUSEKI_ADMIN_PASSWORD}` log in.
|
||||
#
|
||||
# The entrypoint.sh in this image envsubsts the file into /fuseki/shiro.ini
|
||||
# at container start. If you need a different layout, render the substituted
|
||||
# file yourself and bind-mount it onto /fuseki/shiro.ini WITH
|
||||
# FUSEKI_RENDER_SHIRO=false set on the container.
|
||||
#
|
||||
# This integrates with OpenMetadata's authentication
|
||||
|
||||
[main]
|
||||
# Allow anonymous read access, require auth for writes
|
||||
anon = org.apache.shiro.web.filter.authc.AnonymousFilter
|
||||
authcBasic = org.apache.shiro.web.filter.authc.BasicHttpAuthenticationFilter
|
||||
|
||||
# Fuseki 5.x uses Shiro without a session manager configured by default; if
|
||||
# the INI doesn't override the SubjectDAO it ends up trying to use the
|
||||
# default session storage and throws `IllegalStateException: No SessionManager`
|
||||
# on the first authenticated request. Disable session storage so every
|
||||
# request re-authenticates via Basic auth (stateless, REST-style).
|
||||
sessionStorageEvaluator = org.apache.shiro.web.mgt.DefaultWebSessionStorageEvaluator
|
||||
sessionStorageEvaluator.sessionStorageEnabled = false
|
||||
securityManager.subjectDAO.sessionStorageEvaluator = $sessionStorageEvaluator
|
||||
|
||||
# Credentials.
|
||||
#
|
||||
# This file is a TEMPLATE. The entrypoint envsubsts the FUSEKI_ADMIN_PASSWORD
|
||||
# and FUSEKI_OPENMETADATA_PASSWORD variables into the [users] section below
|
||||
# at container start. Defaults applied in the entrypoint:
|
||||
# admin user password = admin
|
||||
# openmetadata user password = openmetadata-secret
|
||||
# (Variable names intentionally NOT written with `$` here so the rendered
|
||||
# file in /fuseki/shiro.ini does not leak the substituted password back
|
||||
# into this comment block.)
|
||||
#
|
||||
# The admin user carries BOTH the `admin` (server-management) and `writer`
|
||||
# (data-mutation) roles so a single credential covers /$/datasets, /$/compact,
|
||||
# /openmetadata/data and /openmetadata/update — Shiro evaluates roles[] as
|
||||
# membership, not permission, so admin's `*` permission alone does NOT satisfy
|
||||
# roles[writer]; explicit membership in both is required.
|
||||
[users]
|
||||
admin = ${FUSEKI_ADMIN_PASSWORD}, admin, writer
|
||||
openmetadata = ${FUSEKI_OPENMETADATA_PASSWORD}, writer
|
||||
|
||||
[roles]
|
||||
admin = *
|
||||
writer = update:*, upload:*, data:*
|
||||
|
||||
[urls]
|
||||
/$/ping = anon
|
||||
/$/stats/* = anon
|
||||
/openmetadata/sparql = anon
|
||||
/openmetadata/query = anon
|
||||
/openmetadata/update = authcBasic, roles[writer]
|
||||
/openmetadata/upload = authcBasic, roles[writer]
|
||||
/openmetadata/data = authcBasic, roles[writer]
|
||||
/** = authcBasic, roles[admin]
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")" || exit
|
||||
|
||||
source ./run_local_docker_common.sh
|
||||
|
||||
run_local_docker_main "$@"
|
||||
Executable
+614
@@ -0,0 +1,614 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
RED='\033[0;31m'
|
||||
RUN_LOCAL_DOCKER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
authorizationToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||
|
||||
helpFunction() {
|
||||
echo ""
|
||||
echo "Usage: $0 -m mode -d database"
|
||||
echo "\t-m Running mode: [ui, no-ui]. Default [ui]\n"
|
||||
echo "\t-d Database: [mysql, postgresql]. Default [mysql]\n"
|
||||
echo "\t-s Skip maven build: [true, false]. Default [false]\n"
|
||||
echo "\t-i Include ingestion: [true, false]. Default [true]\n"
|
||||
echo "\t-x Open JVM debug port on 5005: [true, false]. Default [false]\n"
|
||||
echo "\t-h For usage help\n"
|
||||
echo "\t-r For Cleaning DB Volumes. [true, false]. Default [true]\n"
|
||||
exit 1
|
||||
}
|
||||
|
||||
current_time_ms() {
|
||||
python3 -c "import time; print(int(time.time() * 1000))"
|
||||
}
|
||||
|
||||
run_with_timeout() {
|
||||
local secs=$1
|
||||
shift
|
||||
if command -v timeout &>/dev/null; then
|
||||
timeout "$secs" "$@"
|
||||
elif command -v gtimeout &>/dev/null; then
|
||||
gtimeout "$secs" "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
parse_app_run_status_line() {
|
||||
local payload=$1
|
||||
local payload_shape=$2
|
||||
local threshold_ms=$3
|
||||
local tolerance_ms=$4
|
||||
|
||||
APP_RUN_BODY="$payload" python3 - "$payload_shape" "$threshold_ms" "$tolerance_ms" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
payload_shape = sys.argv[1]
|
||||
threshold = int(sys.argv[2])
|
||||
tolerance = int(sys.argv[3])
|
||||
|
||||
try:
|
||||
payload = json.loads(os.environ["APP_RUN_BODY"])
|
||||
except json.JSONDecodeError:
|
||||
print("invalid")
|
||||
sys.exit(0)
|
||||
|
||||
if payload_shape == "latest":
|
||||
record = payload if isinstance(payload, dict) else None
|
||||
else:
|
||||
records = payload.get("data") if isinstance(payload, dict) else None
|
||||
record = records[0] if records else None
|
||||
|
||||
if not isinstance(record, dict) or not record:
|
||||
print("missing")
|
||||
sys.exit(0)
|
||||
|
||||
timestamp = int(record.get("timestamp") or 0)
|
||||
start_time = int(record.get("startTime") or 0)
|
||||
status = str(record.get("status") or "").lower()
|
||||
execution_time = record.get("executionTime")
|
||||
|
||||
marker = start_time if start_time > 0 else timestamp
|
||||
success_statuses = {"completed", "success"}
|
||||
failure_statuses = {"activeerror", "failed", "stopped"}
|
||||
active_statuses = {"running", "started", "pending", "active", "stopinprogress"}
|
||||
has_execution_time = execution_time not in (None, "", 0, "0")
|
||||
|
||||
if marker + tolerance < threshold:
|
||||
print(f"stale:{status}:{marker}")
|
||||
elif status in success_statuses:
|
||||
print(f"success:{status}:{marker}")
|
||||
elif status in failure_statuses or (has_execution_time and status not in active_statuses):
|
||||
print(f"failure:{status}:{marker}")
|
||||
elif status in active_statuses:
|
||||
print(f"active:{status}:{marker}")
|
||||
else:
|
||||
print(f"seen:{status}:{marker}")
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_app_availability() {
|
||||
local app_name=$1
|
||||
local timeout_seconds=${2:-120}
|
||||
local deadline=$((SECONDS + timeout_seconds))
|
||||
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
local http_code
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
"http://localhost:8585/api/v1/apps/name/${app_name}")
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
echo "✓ App ${app_name} is available"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Waiting for app ${app_name} to become available..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "✗ App ${app_name} did not become available within ${timeout_seconds}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_app_installed() {
|
||||
local app_name=$1
|
||||
local response
|
||||
local http_code
|
||||
local body
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
"http://localhost:8585/api/v1/apps/name/${app_name}")
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
echo "✓ App ${app_name} is already installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$http_code" != "404" ]; then
|
||||
echo "✗ Failed to inspect app ${app_name} (HTTP ${http_code})"
|
||||
echo " Response: ${body}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "App ${app_name} is not installed. Installing it from the marketplace definition..."
|
||||
local marketplace_response
|
||||
marketplace_response=$(curl -s -f \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
"http://localhost:8585/api/v1/apps/marketplace/name/${app_name}") || {
|
||||
echo "✗ Could not fetch marketplace definition for ${app_name}"
|
||||
return 1
|
||||
}
|
||||
|
||||
local create_payload
|
||||
create_payload=$(MARKETPLACE_RESPONSE="$marketplace_response" python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
|
||||
definition = json.loads(os.environ["MARKETPLACE_RESPONSE"])
|
||||
payload = {
|
||||
"name": definition["name"],
|
||||
"displayName": definition.get("displayName"),
|
||||
"description": definition.get("description"),
|
||||
"appConfiguration": definition.get("appConfiguration", {}),
|
||||
"appSchedule": {"scheduleTimeline": "None"},
|
||||
"supportsInterrupt": definition.get("supportsInterrupt", False),
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
PY
|
||||
)
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" --location --request POST 'http://localhost:8585/api/v1/apps' \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$create_payload")
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ] || [ "$http_code" = "409" ]; then
|
||||
wait_for_app_availability "$app_name" 120
|
||||
return $?
|
||||
fi
|
||||
|
||||
echo "✗ Failed to install app ${app_name} (HTTP ${http_code})"
|
||||
echo " Response: ${body}"
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_app_run_completion() {
|
||||
local app_name=$1
|
||||
local trigger_timestamp_ms=$2
|
||||
local timeout_seconds=${3:-${APP_RUN_WAIT_TIMEOUT_SECONDS:-300}}
|
||||
local freshness_tolerance_ms=${APP_RUN_FRESHNESS_TOLERANCE_MS:-5000}
|
||||
local deadline=$((SECONDS + timeout_seconds))
|
||||
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
local latest_run_response
|
||||
local fallback_status_response
|
||||
local http_code
|
||||
local fallback_http_code
|
||||
local body
|
||||
local fallback_body
|
||||
local status_line
|
||||
local fallback_status_line
|
||||
|
||||
latest_run_response=$(curl -s -w "\n%{http_code}" \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
"http://localhost:8585/api/v1/apps/name/${app_name}/runs/latest")
|
||||
http_code=$(printf "%s" "$latest_run_response" | tail -n1)
|
||||
body=$(printf "%s" "$latest_run_response" | sed '$d')
|
||||
|
||||
case "$http_code" in
|
||||
200)
|
||||
status_line=$(parse_app_run_status_line "$body" "latest" "$trigger_timestamp_ms" "$freshness_tolerance_ms")
|
||||
;;
|
||||
204)
|
||||
status_line="missing"
|
||||
;;
|
||||
*)
|
||||
status_line="endpoint_error:${http_code}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$status_line" == stale:* || "$status_line" == "missing" || "$status_line" == "invalid" || "$status_line" == endpoint_error:* ]]; then
|
||||
fallback_status_response=$(curl -s -w "\n%{http_code}" \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
"http://localhost:8585/api/v1/apps/name/${app_name}/status?offset=0&limit=1")
|
||||
fallback_http_code=$(printf "%s" "$fallback_status_response" | tail -n1)
|
||||
fallback_body=$(printf "%s" "$fallback_status_response" | sed '$d')
|
||||
|
||||
case "$fallback_http_code" in
|
||||
200)
|
||||
fallback_status_line=$(parse_app_run_status_line "$fallback_body" "list" "$trigger_timestamp_ms" "$freshness_tolerance_ms")
|
||||
;;
|
||||
204)
|
||||
fallback_status_line="missing"
|
||||
;;
|
||||
*)
|
||||
fallback_status_line="endpoint_error:${fallback_http_code}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$fallback_status_line" != stale:* && "$fallback_status_line" != "missing" && "$fallback_status_line" != "invalid" && "$fallback_status_line" != endpoint_error:* ]]; then
|
||||
status_line="$fallback_status_line"
|
||||
body="$fallback_body"
|
||||
elif [[ "$status_line" == endpoint_error:* && "$fallback_status_line" == endpoint_error:* ]]; then
|
||||
echo "✗ Failed to read run status for ${app_name} from both app run endpoints"
|
||||
echo " runs/latest response: ${body}"
|
||||
echo " status response: ${fallback_body}"
|
||||
return 1
|
||||
elif [[ "$status_line" == "missing" || "$status_line" == "invalid" || "$status_line" == endpoint_error:* ]]; then
|
||||
status_line="$fallback_status_line"
|
||||
body="$fallback_body"
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$status_line" in
|
||||
success:completed:*|success:success:*)
|
||||
echo "✓ ${app_name} completed successfully"
|
||||
return 0
|
||||
;;
|
||||
failure:*)
|
||||
echo "✗ ${app_name} finished with status ${status_line#failure:}"
|
||||
echo " Response: ${body}"
|
||||
return 1
|
||||
;;
|
||||
active:*)
|
||||
echo "Waiting for ${app_name} to finish (${status_line#active:})..."
|
||||
;;
|
||||
seen:*)
|
||||
echo "Waiting for ${app_name}; latest status was ${status_line#seen:}"
|
||||
;;
|
||||
stale:*|missing|invalid)
|
||||
echo "Waiting for a fresh run record for ${app_name}..."
|
||||
;;
|
||||
*)
|
||||
echo "Waiting for ${app_name}; latest status was ${status_line}"
|
||||
;;
|
||||
esac
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "✗ Timed out waiting for ${app_name} to finish within ${timeout_seconds}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
trigger_app_and_wait() {
|
||||
local app_name=$1
|
||||
local payload=${2:-}
|
||||
local timeout_seconds=${3:-${APP_RUN_WAIT_TIMEOUT_SECONDS:-300}}
|
||||
local trigger_timestamp_ms
|
||||
local response
|
||||
local http_code
|
||||
local body
|
||||
|
||||
trigger_timestamp_ms=$(current_time_ms)
|
||||
if [ -n "$payload" ]; then
|
||||
response=$(curl -s -w "\n%{http_code}" --location --request POST "http://localhost:8585/api/v1/apps/trigger/${app_name}" \
|
||||
--header "Authorization: Bearer $authorizationToken" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$payload")
|
||||
else
|
||||
response=$(curl -s -w "\n%{http_code}" --location --request POST "http://localhost:8585/api/v1/apps/trigger/${app_name}" \
|
||||
--header "Authorization: Bearer $authorizationToken")
|
||||
fi
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" != "200" ] && [ "$http_code" != "201" ] && [ "$http_code" != "202" ]; then
|
||||
echo "✗ Failed to trigger ${app_name} (HTTP ${http_code})"
|
||||
echo " Response: ${body}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
wait_for_app_run_completion "$app_name" "$trigger_timestamp_ms" "$timeout_seconds"
|
||||
}
|
||||
|
||||
run_local_docker_main() {
|
||||
local mode database skipMaven includeIngestion debugOM cleanDbVolumes
|
||||
local COMPOSE_FILE DB_SERVICE SEARCH_SERVICE
|
||||
local response http_code body
|
||||
local validation_timeout_seconds sample_data_validation_failed
|
||||
local AIRFLOW_ACCESS_TOKEN IMPORT_ERRORS LOGICAL_DATE
|
||||
local extra_compose_file
|
||||
local -a COMPOSE_ARGS additional_up_services
|
||||
|
||||
OPTIND=1
|
||||
while getopts "m:d:s:i:x:r:h" opt; do
|
||||
case "$opt" in
|
||||
m ) mode="$OPTARG" ;;
|
||||
d ) database="$OPTARG" ;;
|
||||
s ) skipMaven="$OPTARG" ;;
|
||||
i ) includeIngestion="$OPTARG" ;;
|
||||
x ) debugOM="$OPTARG" ;;
|
||||
r ) cleanDbVolumes="$OPTARG" ;;
|
||||
h ) helpFunction ;;
|
||||
? ) helpFunction ;;
|
||||
esac
|
||||
done
|
||||
|
||||
mode="${mode:=ui}"
|
||||
database="${database:=mysql}"
|
||||
skipMaven="${skipMaven:=false}"
|
||||
includeIngestion="${includeIngestion:=true}"
|
||||
debugOM="${debugOM:=false}"
|
||||
cleanDbVolumes="${cleanDbVolumes:=true}"
|
||||
export APP_RUN_WAIT_TIMEOUT_SECONDS="${APP_RUN_WAIT_TIMEOUT_SECONDS:-300}"
|
||||
|
||||
echo "Running local docker using mode [$mode] database [$database] and skipping maven build [$skipMaven] with cleanDB as [$cleanDbVolumes] and including ingestion [$includeIngestion]"
|
||||
|
||||
cd "$RUN_LOCAL_DOCKER_DIR/.." || exit 1
|
||||
|
||||
echo "Stopping any previous Local Docker Containers"
|
||||
docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans
|
||||
docker compose -f docker/development/docker-compose.yml down --remove-orphans
|
||||
if [[ -n "${OM_EXTRA_COMPOSE_FILES:-}" ]]; then
|
||||
for extra_compose_file in $OM_EXTRA_COMPOSE_FILES; do
|
||||
docker compose -f docker/development/docker-compose-postgres.yml -f "$extra_compose_file" down --remove-orphans
|
||||
docker compose -f docker/development/docker-compose.yml -f "$extra_compose_file" down --remove-orphans
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ $skipMaven == "false" ]]; then
|
||||
if [[ $mode == "no-ui" ]]; then
|
||||
echo "Maven Build - Skipping Tests and UI"
|
||||
mvn -DskipTests -DonlyBackend clean package -pl !openmetadata-ui
|
||||
else
|
||||
echo "Maven Build - Skipping Tests"
|
||||
mvn -DskipTests clean package
|
||||
fi
|
||||
else
|
||||
echo "Skipping Maven Build"
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to run Maven build!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $debugOM == "true" ]]; then
|
||||
export OPENMETADATA_DEBUG=true
|
||||
fi
|
||||
|
||||
if [[ $cleanDbVolumes == "true" ]]; then
|
||||
if [[ -d "$PWD/docker/development/docker-volume/" ]]; then
|
||||
if ! rm -rf "$PWD/docker/development/docker-volume"; then
|
||||
if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then
|
||||
sudo rm -rf "$PWD/docker/development/docker-volume"
|
||||
else
|
||||
echo "Warning: failed to remove $PWD/docker/development/docker-volume; continuing may reuse stale database state"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $includeIngestion == "true" ]]; then
|
||||
if [[ $VIRTUAL_ENV == "" ]]; then
|
||||
echo "Please Use Virtual Environment and make sure to generate Pydantic Models"
|
||||
else
|
||||
echo "Generating Pydantic Models"
|
||||
make install_dev generate
|
||||
fi
|
||||
else
|
||||
echo "Skipping Pydantic Models generation (ingestion disabled)"
|
||||
fi
|
||||
|
||||
echo "Starting Local Docker Containers"
|
||||
|
||||
if [[ $database == "postgresql" ]]; then
|
||||
COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
|
||||
DB_SERVICE="postgresql"
|
||||
SEARCH_SERVICE="opensearch"
|
||||
elif [[ $database == "mysql" ]]; then
|
||||
COMPOSE_FILE="docker/development/docker-compose.yml"
|
||||
DB_SERVICE="mysql"
|
||||
SEARCH_SERVICE="elasticsearch"
|
||||
else
|
||||
echo "Invalid database type: $database"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COMPOSE_ARGS=(-f "$COMPOSE_FILE")
|
||||
if [[ -n "${OM_EXTRA_COMPOSE_FILES:-}" ]]; then
|
||||
for extra_compose_file in $OM_EXTRA_COMPOSE_FILES; do
|
||||
COMPOSE_ARGS+=(-f "$extra_compose_file")
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ $includeIngestion == "true" ]]; then
|
||||
echo "Building all services including ingestion (dependency: ${INGESTION_DEPENDENCY:-all})"
|
||||
docker compose "${COMPOSE_ARGS[@]}" build --build-arg INGESTION_DEPENDENCY="${INGESTION_DEPENDENCY:-all}" && \
|
||||
docker compose "${COMPOSE_ARGS[@]}" up -d
|
||||
else
|
||||
echo "Building services without ingestion"
|
||||
docker compose "${COMPOSE_ARGS[@]}" build $SEARCH_SERVICE $DB_SERVICE execute-migrate-all openmetadata-server || exit 1
|
||||
if [[ -n "${OM_ADDITIONAL_UP_SERVICES:-}" ]]; then
|
||||
for extra_compose_file in $OM_ADDITIONAL_UP_SERVICES; do
|
||||
additional_up_services+=("$extra_compose_file")
|
||||
done
|
||||
fi
|
||||
docker compose "${COMPOSE_ARGS[@]}" up -d "${additional_up_services[@]}" $SEARCH_SERVICE $DB_SERVICE execute-migrate-all openmetadata-server
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to start Docker instances!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
until curl -s -f "http://localhost:9200/_cat/indices/openmetadata_team_search_index"; do
|
||||
echo 'Checking if Elastic Search instance is up...\n'
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [[ $includeIngestion == "true" ]]; then
|
||||
get_airflow_token() {
|
||||
local token_response
|
||||
local access_token
|
||||
|
||||
token_response=$(curl -s -X POST 'http://localhost:8080/auth/token' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"username": "admin", "password": "admin"}')
|
||||
|
||||
access_token=$(echo "$token_response" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('access_token', ''))" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$access_token" ]; then
|
||||
echo "✗ Failed to get access token" >&2
|
||||
echo " Response: ${token_response}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$access_token"
|
||||
}
|
||||
|
||||
echo "Waiting for Airflow API to be ready..."
|
||||
until AIRFLOW_ACCESS_TOKEN=$(get_airflow_token) 2>/dev/null && [ -n "$AIRFLOW_ACCESS_TOKEN" ]; do
|
||||
echo 'Checking if Airflow API is reachable...'
|
||||
sleep 5
|
||||
done
|
||||
echo "✓ Airflow API is ready, token obtained"
|
||||
|
||||
echo "Checking if Sample Data DAG is available..."
|
||||
until curl -s -f -H "Authorization: Bearer $AIRFLOW_ACCESS_TOKEN" "http://localhost:8080/api/v2/dags/sample_data" >/dev/null 2>&1; do
|
||||
IMPORT_ERRORS=$(curl -s -H "Authorization: Bearer $AIRFLOW_ACCESS_TOKEN" "http://localhost:8080/api/v2/importErrors" 2>/dev/null)
|
||||
if [ -n "$IMPORT_ERRORS" ]; then
|
||||
echo "$IMPORT_ERRORS" | grep "/airflow_sample_data.py" > /dev/null 2>&1
|
||||
if [ "$?" == "0" ]; then
|
||||
echo -e "${RED}Airflow found an error importing \`sample_data\` DAG"
|
||||
echo "$IMPORT_ERRORS" | python3 -c "import sys, json; data=json.load(sys.stdin); [print(json.dumps(e, indent=2)) for e in data.get('import_errors', []) if e.get('filename', '').endswith('airflow_sample_data.py')]" 2>/dev/null || echo "$IMPORT_ERRORS"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo 'Checking if Sample Data DAG is reachable...'
|
||||
sleep 5
|
||||
AIRFLOW_ACCESS_TOKEN=$(get_airflow_token) 2>/dev/null
|
||||
done
|
||||
echo "✓ Sample Data DAG is available"
|
||||
fi
|
||||
|
||||
until curl -s -f --header "Authorization: Bearer $authorizationToken" "http://localhost:8585/api/v1/tables"; do
|
||||
echo 'Checking if OM Server is reachable...\n'
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [[ $includeIngestion == "true" ]]; then
|
||||
unpause_dag() {
|
||||
local dag_id=$1
|
||||
|
||||
echo "Unpausing DAG: ${dag_id}"
|
||||
if [ -z "$AIRFLOW_ACCESS_TOKEN" ]; then
|
||||
AIRFLOW_ACCESS_TOKEN=$(get_airflow_token)
|
||||
if [ -z "$AIRFLOW_ACCESS_TOKEN" ]; then
|
||||
return 1
|
||||
fi
|
||||
echo "✓ OAuth token obtained"
|
||||
fi
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" --location --request PATCH "http://localhost:8080/api/v2/dags/${dag_id}" \
|
||||
--header "Authorization: Bearer $AIRFLOW_ACCESS_TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw '{"is_paused": false}')
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
body=$(echo "$response" | sed '$d')
|
||||
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
echo "✓ Successfully unpaused ${dag_id}"
|
||||
else
|
||||
echo "✗ Failed to unpause ${dag_id} (HTTP ${http_code})"
|
||||
echo " Response: ${body}"
|
||||
if [ "$http_code" = "401" ]; then
|
||||
echo " Refreshing token and retrying..."
|
||||
AIRFLOW_ACCESS_TOKEN=$(get_airflow_token)
|
||||
if [ -n "$AIRFLOW_ACCESS_TOKEN" ]; then
|
||||
response=$(curl -s -w "\n%{http_code}" --location --request PATCH "http://localhost:8080/api/v2/dags/${dag_id}" \
|
||||
--header "Authorization: Bearer $AIRFLOW_ACCESS_TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw '{"is_paused": false}')
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
echo "✓ Successfully unpaused ${dag_id} after retry"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
unpause_dag "sample_data"
|
||||
unpause_dag "extended_sample_data"
|
||||
|
||||
echo "Triggering sample_data DAG..."
|
||||
LOGICAL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:8080/api/v2/dags/sample_data/dagRuns" \
|
||||
--header "Authorization: Bearer $AIRFLOW_ACCESS_TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "{\"logical_date\": \"$LOGICAL_DATE\"}")
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
|
||||
echo "✓ Successfully triggered sample_data DAG"
|
||||
else
|
||||
echo "⚠ Could not trigger sample_data DAG (HTTP ${http_code})"
|
||||
echo " Response: $(echo "$response" | sed '$d')"
|
||||
echo " Note: DAG may run automatically on schedule"
|
||||
fi
|
||||
|
||||
echo 'Validate sample data DAG...'
|
||||
sleep 5
|
||||
make install
|
||||
|
||||
echo "Running DAG validation (this may take a few minutes)..."
|
||||
sample_data_validation_failed=false
|
||||
validation_timeout_seconds="${VALIDATION_TIMEOUT_SECONDS:-300}"
|
||||
|
||||
run_with_timeout "$validation_timeout_seconds" python docker/validate_compose.py || {
|
||||
local exit_code=$?
|
||||
sample_data_validation_failed=true
|
||||
if [ $exit_code -eq 124 ]; then
|
||||
echo "⚠ Warning: DAG validation timed out after ${validation_timeout_seconds} seconds"
|
||||
echo " The DAG may still be running. Check Airflow UI at http://localhost:8080"
|
||||
else
|
||||
echo "⚠ Warning: DAG validation failed with exit code $exit_code"
|
||||
fi
|
||||
echo " Continuing with remaining setup..."
|
||||
}
|
||||
|
||||
if [[ "${STRICT_DAG_VALIDATION:-false}" == "true" && "$sample_data_validation_failed" == "true" ]]; then
|
||||
echo "✗ Startup requires sample data ingestion to complete before continuing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
unpause_dag "sample_usage"
|
||||
sleep 5
|
||||
unpause_dag "index_metadata"
|
||||
sleep 2
|
||||
unpause_dag "sample_lineage"
|
||||
else
|
||||
echo "Skipping Airflow DAG setup (ingestion disabled)"
|
||||
fi
|
||||
|
||||
echo "✔running reindexing"
|
||||
ensure_app_installed "SearchIndexingApplication"
|
||||
if ! trigger_app_and_wait "SearchIndexingApplication" "" "$APP_RUN_WAIT_TIMEOUT_SECONDS"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tput setaf 2
|
||||
echo "✔ OpenMetadata is up and running"
|
||||
}
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")" || exit
|
||||
|
||||
helpFunction() {
|
||||
echo ""
|
||||
echo "Usage: $0 [run_local_docker.sh args]"
|
||||
echo "\t-f Start Fuseki for RDF support: [true, false]. Default [true]"
|
||||
echo "\t-h For usage help"
|
||||
exit 1
|
||||
}
|
||||
|
||||
startFuseki=true
|
||||
filtered_args=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-f)
|
||||
if [[ $# -lt 2 ]]; then
|
||||
helpFunction
|
||||
fi
|
||||
startFuseki="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h)
|
||||
helpFunction
|
||||
;;
|
||||
*)
|
||||
filtered_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $startFuseki == "true" ]]; then
|
||||
export RDF_ENABLED=true
|
||||
export RDF_AUTO_REINDEX=true
|
||||
export RDF_STORAGE_TYPE="${RDF_STORAGE_TYPE:-FUSEKI}"
|
||||
export RDF_ENDPOINT="${RDF_ENDPOINT:-http://fuseki:3030/openmetadata}"
|
||||
export RDF_REMOTE_USERNAME="${RDF_REMOTE_USERNAME:-admin}"
|
||||
export RDF_REMOTE_PASSWORD="${RDF_REMOTE_PASSWORD:-admin}"
|
||||
export RDF_BASE_URI="${RDF_BASE_URI:-https://open-metadata.org/}"
|
||||
export RDF_DATASET="${RDF_DATASET:-openmetadata}"
|
||||
# RDF listeners slow down sample-data ingestion enough that the default 5-minute
|
||||
# validation window is too aggressive for CI.
|
||||
export VALIDATE_COMPOSE_MAX_RETRIES="${VALIDATE_COMPOSE_MAX_RETRIES:-60}"
|
||||
export VALIDATE_COMPOSE_DAG_RUN_RETRIES="${VALIDATE_COMPOSE_DAG_RUN_RETRIES:-120}"
|
||||
export VALIDATE_COMPOSE_RETRY_INTERVAL_SECONDS="${VALIDATE_COMPOSE_RETRY_INTERVAL_SECONDS:-10}"
|
||||
export VALIDATE_COMPOSE_DAG_RUN_POLL_SECONDS="${VALIDATE_COMPOSE_DAG_RUN_POLL_SECONDS:-5}"
|
||||
export VALIDATION_TIMEOUT_SECONDS="${VALIDATION_TIMEOUT_SECONDS:-900}"
|
||||
export APP_RUN_WAIT_TIMEOUT_SECONDS="${APP_RUN_WAIT_TIMEOUT_SECONDS:-900}"
|
||||
export STRICT_DAG_VALIDATION=true
|
||||
export OM_EXTRA_COMPOSE_FILES="docker/development/docker-compose-fuseki.yml"
|
||||
export OM_ADDITIONAL_UP_SERVICES="fuseki"
|
||||
else
|
||||
export RDF_ENABLED=false
|
||||
export RDF_AUTO_REINDEX=false
|
||||
unset RDF_STORAGE_TYPE
|
||||
unset RDF_ENDPOINT
|
||||
unset RDF_REMOTE_USERNAME
|
||||
unset RDF_REMOTE_PASSWORD
|
||||
unset RDF_BASE_URI
|
||||
unset RDF_DATASET
|
||||
unset VALIDATE_COMPOSE_MAX_RETRIES
|
||||
unset VALIDATE_COMPOSE_DAG_RUN_RETRIES
|
||||
unset VALIDATE_COMPOSE_RETRY_INTERVAL_SECONDS
|
||||
unset VALIDATE_COMPOSE_DAG_RUN_POLL_SECONDS
|
||||
export VALIDATION_TIMEOUT_SECONDS="${VALIDATION_TIMEOUT_SECONDS:-300}"
|
||||
export APP_RUN_WAIT_TIMEOUT_SECONDS="${APP_RUN_WAIT_TIMEOUT_SECONDS:-300}"
|
||||
unset STRICT_DAG_VALIDATION
|
||||
unset OM_EXTRA_COMPOSE_FILES
|
||||
unset OM_ADDITIONAL_UP_SERVICES
|
||||
fi
|
||||
|
||||
source ./run_local_docker_common.sh
|
||||
|
||||
run_local_docker_main "${filtered_args[@]}"
|
||||
|
||||
if [[ $startFuseki != "true" || $RDF_AUTO_REINDEX != "true" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ensure_app_installed "RdfIndexApp"
|
||||
echo "✔running RDF reindexing"
|
||||
trigger_app_and_wait "RdfIndexApp" '{
|
||||
"entities": [],
|
||||
"recreateIndex": true,
|
||||
"batchSize": 100,
|
||||
"useDistributedIndexing": true,
|
||||
"partitionSize": 10000
|
||||
}' "$APP_RUN_WAIT_TIMEOUT_SECONDS"
|
||||
|
||||
tput setaf 2
|
||||
echo "✔ RDF/Knowledge Graph support is enabled"
|
||||
echo " - Fuseki UI: http://localhost:3030"
|
||||
echo " - SPARQL endpoint: http://localhost:3030/openmetadata/sparql"
|
||||
@@ -0,0 +1,55 @@
|
||||
# Copyright 2024 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# ── Stage 1: build Storybook static assets ──────────────────────────────────
|
||||
# Build context must be the repo root so both the source tree and this config
|
||||
# file are reachable. The nightly workflow calls:
|
||||
# docker build -f docker/storybook/Dockerfile .
|
||||
FROM node:22.17-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy lockfile + manifest first so the layer is cached when only source changes.
|
||||
COPY openmetadata-ui-core-components/src/main/resources/ui/package.json \
|
||||
openmetadata-ui-core-components/src/main/resources/ui/yarn.lock ./
|
||||
|
||||
# Install all dependencies (dev deps are required for the Storybook build).
|
||||
RUN yarn install --frozen-lockfile --network-timeout 120000
|
||||
|
||||
# Copy the rest of the source tree.
|
||||
COPY openmetadata-ui-core-components/src/main/resources/ui/ ./
|
||||
|
||||
# Produce the static site in storybook-static/.
|
||||
RUN yarn build-storybook
|
||||
|
||||
# ── Stage 2: serve with Node ─────────────────────────────────────────────────
|
||||
FROM node:22.17-alpine AS server
|
||||
|
||||
# `serve` is the static file server Storybook recommends; pin the major so the
|
||||
# CLI flags stay stable across nightly rebuilds.
|
||||
RUN npm install -g serve@14
|
||||
|
||||
COPY --from=builder /app/storybook-static /app/storybook-static
|
||||
|
||||
# Storybook 8's manager requests the preview at /iframe (no .html extension).
|
||||
# serve's --single SPA fallback would intercept that path (no matching file on
|
||||
# disk) and return index.html — the manager shell — into the preview iframe,
|
||||
# so components never render. This rewrite rule maps /iframe → /iframe.html
|
||||
# before the fallback runs, giving the preview iframe the correct content.
|
||||
RUN printf '{"rewrites":[{"source":"/iframe","destination":"/iframe.html"}]}' \
|
||||
> /app/serve.json
|
||||
|
||||
# Drop to the built-in unprivileged user — running a public server as root is unnecessary.
|
||||
USER node
|
||||
|
||||
EXPOSE 6006
|
||||
|
||||
CMD ["serve", "/app/storybook-static", "-l", "6006", "-c", "/app/serve.json"]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Allowlist-style ignore for docker/storybook/Dockerfile.
|
||||
# Excludes the entire monorepo by default, then re-includes only the files the
|
||||
# Dockerfile actually COPYs, keeping the build context to a few MB instead of
|
||||
# hundreds of MB.
|
||||
|
||||
# Exclude everything.
|
||||
**
|
||||
|
||||
# Re-include the component library source (each parent dir must be listed explicitly).
|
||||
!openmetadata-ui-core-components/
|
||||
!openmetadata-ui-core-components/src/
|
||||
!openmetadata-ui-core-components/src/main/
|
||||
!openmetadata-ui-core-components/src/main/resources/
|
||||
!openmetadata-ui-core-components/src/main/resources/ui/
|
||||
!openmetadata-ui-core-components/src/main/resources/ui/**
|
||||
|
||||
# Re-exclude build artifacts that should never enter the image.
|
||||
openmetadata-ui-core-components/src/main/resources/ui/node_modules
|
||||
openmetadata-ui-core-components/src/main/resources/ui/storybook-static
|
||||
openmetadata-ui-core-components/src/main/resources/ui/dist
|
||||
@@ -0,0 +1,12 @@
|
||||
# Local smoke-test compose. Run from the repo root:
|
||||
# docker compose -f docker/storybook/docker-compose.yml up --build
|
||||
#
|
||||
# Then open http://localhost:6006
|
||||
services:
|
||||
storybook:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/storybook/Dockerfile
|
||||
ports:
|
||||
- "6006:6006"
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,188 @@
|
||||
import os
|
||||
import time
|
||||
from pprint import pprint
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import requests
|
||||
from metadata.utils.logger import log_ansi_encoded_string
|
||||
|
||||
|
||||
HEADER_JSON = {"Content-Type": "application/json"}
|
||||
REQUESTS_TIMEOUT = 60 * 5
|
||||
AIRFLOW_URL = "http://localhost:8080"
|
||||
USERNAME = "admin"
|
||||
PASSWORD = "admin"
|
||||
|
||||
_access_token: Optional[str] = None
|
||||
_last_dag_logs_supported: Optional[bool] = None
|
||||
|
||||
|
||||
def get_env_int(name: str, default: int) -> int:
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
return default
|
||||
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
log_ansi_encoded_string(
|
||||
message=f"Invalid integer for {name}: {value}. Falling back to {default}."
|
||||
)
|
||||
return default
|
||||
|
||||
|
||||
def get_access_token() -> str:
|
||||
"""Get OAuth access token for Airflow 3.x API."""
|
||||
global _access_token
|
||||
|
||||
if _access_token:
|
||||
return _access_token
|
||||
|
||||
response = requests.post(
|
||||
f"{AIRFLOW_URL}/auth/token",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json={"username": USERNAME, "password": PASSWORD},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
data = response.json()
|
||||
_access_token = data.get("access_token")
|
||||
return _access_token
|
||||
else:
|
||||
raise Exception(f"Failed to get access token: {response.status_code} - {response.text}")
|
||||
|
||||
|
||||
def get_auth_headers() -> dict:
|
||||
"""Get authorization headers with Bearer token."""
|
||||
token = get_access_token()
|
||||
return {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
|
||||
def get_last_run_info() -> Tuple[str, str]:
|
||||
"""
|
||||
Make sure we can pick up the latest run info
|
||||
"""
|
||||
max_retries = get_env_int("VALIDATE_COMPOSE_DAG_RUN_RETRIES", 30)
|
||||
poll_interval_seconds = get_env_int("VALIDATE_COMPOSE_DAG_RUN_POLL_SECONDS", 5)
|
||||
retries = 0
|
||||
|
||||
while retries < max_retries:
|
||||
try:
|
||||
res = requests.get(
|
||||
f"{AIRFLOW_URL}/api/v2/dags/sample_data/dagRuns",
|
||||
headers=get_auth_headers(),
|
||||
timeout=REQUESTS_TIMEOUT
|
||||
)
|
||||
res.raise_for_status()
|
||||
runs = res.json()
|
||||
dag_runs = runs.get("dag_runs", [])
|
||||
|
||||
if dag_runs and len(dag_runs) > 0:
|
||||
# Sort by logical_date descending to get the latest run
|
||||
dag_runs_sorted = sorted(
|
||||
dag_runs,
|
||||
key=lambda x: x.get("logical_date", ""),
|
||||
reverse=True
|
||||
)
|
||||
dag_run = dag_runs_sorted[0]
|
||||
dag_run_id = dag_run.get("dag_run_id")
|
||||
state = dag_run.get("state", "").lower()
|
||||
|
||||
if dag_run_id:
|
||||
log_ansi_encoded_string(
|
||||
message=f"Found DAG run: {dag_run_id} with state: {state}"
|
||||
)
|
||||
return dag_run_id, state
|
||||
else:
|
||||
log_ansi_encoded_string(message="No DAG runs found yet, waiting...")
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
global _access_token
|
||||
_access_token = None
|
||||
log_ansi_encoded_string(message=f"Error getting DAG runs: {e}")
|
||||
|
||||
time.sleep(poll_interval_seconds)
|
||||
retries += 1
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def print_last_run_logs() -> None:
|
||||
"""
|
||||
Show the logs
|
||||
"""
|
||||
global _last_dag_logs_supported
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{AIRFLOW_URL}/api/v2/openmetadata/last_dag_logs?dag_id=sample_data&task_id=ingest_using_recipe",
|
||||
headers=get_auth_headers(),
|
||||
timeout=REQUESTS_TIMEOUT
|
||||
)
|
||||
|
||||
if response.status_code == 404:
|
||||
if _last_dag_logs_supported is not False:
|
||||
log_ansi_encoded_string(
|
||||
message="Airflow last_dag_logs route is unavailable. Skipping task log fetch."
|
||||
)
|
||||
_last_dag_logs_supported = False
|
||||
return
|
||||
|
||||
response.raise_for_status()
|
||||
_last_dag_logs_supported = True
|
||||
pprint(response.text)
|
||||
except Exception as e:
|
||||
log_ansi_encoded_string(message=f"Could not fetch logs: {e}")
|
||||
|
||||
|
||||
def main():
|
||||
max_retries = get_env_int("VALIDATE_COMPOSE_MAX_RETRIES", 15)
|
||||
retry_interval_seconds = get_env_int("VALIDATE_COMPOSE_RETRY_INTERVAL_SECONDS", 10)
|
||||
retries = 0
|
||||
|
||||
while retries < max_retries:
|
||||
dag_run_id, state = get_last_run_info()
|
||||
|
||||
if not dag_run_id:
|
||||
log_ansi_encoded_string(
|
||||
message="Waiting for DAG run to start...",
|
||||
)
|
||||
time.sleep(retry_interval_seconds)
|
||||
retries += 1
|
||||
continue
|
||||
|
||||
if state == "success":
|
||||
log_ansi_encoded_string(message=f"DAG run: [{dag_run_id}, {state}]")
|
||||
print_last_run_logs()
|
||||
log_ansi_encoded_string(message="Sample data ingestion completed successfully!")
|
||||
break
|
||||
elif state in ["running", "queued"]:
|
||||
log_ansi_encoded_string(
|
||||
message=f"DAG run [{dag_run_id}] is {state}. Waiting for completion...",
|
||||
)
|
||||
print_last_run_logs()
|
||||
time.sleep(retry_interval_seconds)
|
||||
retries += 1
|
||||
elif state == "failed":
|
||||
log_ansi_encoded_string(message=f"DAG run [{dag_run_id}] FAILED!")
|
||||
print_last_run_logs()
|
||||
raise Exception(f"Sample data ingestion failed. DAG run state: {state}")
|
||||
else:
|
||||
log_ansi_encoded_string(
|
||||
message=f"Waiting for sample data ingestion. Current state: {state}",
|
||||
)
|
||||
print_last_run_logs()
|
||||
time.sleep(retry_interval_seconds)
|
||||
retries += 1
|
||||
|
||||
if retries == max_retries:
|
||||
raise Exception("Max retries exceeded. Sample data ingestion was not successful.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user