chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
+20
@@ -0,0 +1,20 @@
|
||||
-- Create tables for tracking server migrations
|
||||
-- This migration runs before all other migrations to ensure migration tracking tables exist
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
|
||||
installed_rank BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
version VARCHAR(256) NOT NULL,
|
||||
migrationFileName VARCHAR(256) NOT NULL,
|
||||
checksum VARCHAR(256) NOT NULL,
|
||||
installed_on TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
metrics JSON,
|
||||
PRIMARY KEY (version),
|
||||
UNIQUE KEY installed_rank (installed_rank)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
|
||||
version VARCHAR(256) NOT NULL,
|
||||
sqlStatement VARCHAR(10000) NOT NULL,
|
||||
checksum VARCHAR(256) PRIMARY KEY,
|
||||
executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
--
|
||||
-- Table that captures all the relationships between entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_relationship (
|
||||
fromId VARCHAR(36) NOT NULL, -- ID of the from entity
|
||||
toId VARCHAR(36) NOT NULL, -- ID of the to entity
|
||||
fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity
|
||||
toEntity VARCHAR(256) NOT NULL, -- Type name of to entity
|
||||
relation TINYINT NOT NULL,
|
||||
jsonSchema VARCHAR(256), -- Schema used for generating JSON
|
||||
json JSON, -- JSON payload with additional information
|
||||
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
INDEX edge_index (fromId, toId, relation),
|
||||
INDEX from_index (fromId, relation),
|
||||
INDEX to_index (toId, relation),
|
||||
PRIMARY KEY (fromId, toId, relation)
|
||||
);
|
||||
|
||||
--
|
||||
-- Table that captures all the relationships between field of an entity to a field of another entity
|
||||
-- Example - table1.column1 (fromFQN of type table.columns.column) is joined with(relation)
|
||||
-- table2.column8 (toFQN of type table.columns.column)
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS field_relationship (
|
||||
fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
|
||||
toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
|
||||
fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
|
||||
toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
|
||||
relation TINYINT NOT NULL,
|
||||
jsonSchema VARCHAR(256), -- Schema used for generating JSON
|
||||
json JSON, -- JSON payload with additional information
|
||||
INDEX from_index (fromFQN, relation),
|
||||
INDEX to_index (toFQN, relation),
|
||||
PRIMARY KEY (fromFQN, toFQN, relation)
|
||||
);
|
||||
|
||||
--
|
||||
-- Used for storing additional metadata for an entity
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_extension (
|
||||
id VARCHAR(36) NOT NULL, -- ID of the from entity
|
||||
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
|
||||
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id, extension)
|
||||
);
|
||||
|
||||
--
|
||||
-- Service entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS dbservice_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messaging_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dashboard_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pipeline_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS storage_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
--
|
||||
-- Data entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS database_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS database_schema_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS table_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metric_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dashboard_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pipeline_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS topic_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chart_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS location_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL ,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- Feed related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS thread_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityId') NOT NULL,
|
||||
entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL,
|
||||
assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.addressedTo'),
|
||||
json JSON NOT NULL,
|
||||
createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.threadTs') STORED NOT NULL,
|
||||
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.createdBy') STORED NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
resolved BOOLEAN GENERATED ALWAYS AS (json -> '$.resolved'),
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
--
|
||||
-- Policies related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS policy_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- Ingestion related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
timestamp BIGINT,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- User, Team, and bots
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS team_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
email VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.email') NOT NULL,
|
||||
deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> '$.deactivated'),
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bot_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS role_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'),
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
--
|
||||
-- Usage table where usage for all the entities is captured
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_usage (
|
||||
id VARCHAR(36) NOT NULL, -- Unique id of the entity
|
||||
entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
|
||||
usageDate DATE, -- date corresponding to the usage
|
||||
count1 INT, -- total daily count of use on usageDate
|
||||
count7 INT, -- rolling count of last 7 days going back from usageDate
|
||||
count30 INT, -- rolling count of last 30 days going back from usageDate
|
||||
percentile1 INT, -- percentile rank with in same entity for given usage date
|
||||
percentile7 INT, -- percentile rank with in same entity for last 7 days of usage
|
||||
percentile30 INT, -- percentile rank with in same entity for last 30 days of usage
|
||||
UNIQUE (usageDate, id)
|
||||
);
|
||||
|
||||
--
|
||||
-- Tag related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS tag_category (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL, -- JSON stores category information and does not store children
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE (name) -- Unique tag category name
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL, -- JSON stores all tag attributes and does not store children
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag_usage (
|
||||
source TINYINT NOT NULL, -- Source of the tag label
|
||||
tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag
|
||||
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
|
||||
labelType TINYINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived
|
||||
state TINYINT NOT NULL, -- State of tagging: suggested or confirmed
|
||||
UNIQUE (source, tagFQN, targetFQN)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS change_event (
|
||||
eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
|
||||
entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL,
|
||||
userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL,
|
||||
eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
INDEX event_type_index (eventType),
|
||||
INDEX entity_type_index (entityType),
|
||||
INDEX event_time_index (eventTime)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS webhook_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS glossary_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS glossary_term_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
CREATE TABLE IF NOT EXISTS type_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
category VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.category') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
ADD status VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.status') NOT NULL,
|
||||
DROP COLUMN deleted;
|
||||
|
||||
ALTER TABLE entity_relationship
|
||||
DROP INDEX edge_index;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mlmodel_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
UPDATE thread_entity SET json = JSON_SET(json, '$.type', 'Conversation', '$.reactions', JSON_ARRAY());
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD type VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.type'),
|
||||
ADD taskId INT UNSIGNED GENERATED ALWAYS AS (json ->> '$.task.id'),
|
||||
ADD taskStatus VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.task.status'),
|
||||
ADD taskAssignees JSON GENERATED ALWAYS AS (json -> '$.task.assignees'),
|
||||
ADD CONSTRAINT task_id_constraint UNIQUE(taskId),
|
||||
ADD INDEX thread_type_index (type),
|
||||
ADD INDEX task_status_index (taskStatus),
|
||||
ADD INDEX created_by_index (createdBy),
|
||||
ADD INDEX updated_at_index (updatedAt);
|
||||
|
||||
CREATE TABLE task_sequence (id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id));
|
||||
INSERT INTO task_sequence VALUES (0);
|
||||
|
||||
|
||||
DELETE from ingestion_pipeline_entity where 1=1;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.database'),
|
||||
'$.connection.config.databaseSchema',
|
||||
JSON_EXTRACT(json, '$.connection.config.database')
|
||||
) where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle');
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.database'
|
||||
,'$.connection.config.username'
|
||||
,'$.connection.config.projectId'
|
||||
,'$.connection.config.enablePolicyTagImport')
|
||||
WHERE serviceType = 'BigQuery';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.database')
|
||||
WHERE serviceType in ('Athena','Databricks');
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler', '$.connection.config.pipelineServiceName')
|
||||
WHERE serviceType = 'Glue';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.dbServiceName')
|
||||
WHERE serviceType in ('Metabase','Superset','Tableau');
|
||||
|
||||
DELETE FROM pipeline_service_entity WHERE 1=1;
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
DELETE from ingestion_pipeline_entity where 1=1;
|
||||
DELETE from entity_relationship where toEntity = 'ingestionPipeline';
|
||||
|
||||
-- 0.10 had empty FQNs for services, users and teams
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE messaging_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE storage_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE team_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
|
||||
UPDATE user_entity
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.fullyQualifiedName',
|
||||
JSON_EXTRACT(json, '$.name')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
UPDATE team_entity
|
||||
SET json = JSON_INSERT(json, '$.teamType', 'Group');
|
||||
|
||||
ALTER TABLE team_entity
|
||||
ADD teamType VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.teamType') NOT NULL;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.database')
|
||||
WHERE serviceType = 'DynamoDB';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions')
|
||||
WHERE serviceType = 'DeltaLake';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler')
|
||||
WHERE serviceType = 'DeltaLake';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.username'),
|
||||
'$.connection.config.clientId',
|
||||
JSON_EXTRACT(json, '$.connection.config.username')
|
||||
)
|
||||
WHERE serviceType = 'Looker';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.password'),
|
||||
'$.connection.config.clientSecret',
|
||||
JSON_EXTRACT(json, '$.connection.config.password')
|
||||
)
|
||||
WHERE serviceType = 'Looker';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.env')
|
||||
WHERE serviceType = 'Looker';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_definition (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_suite (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_case (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
entityFQN VARCHAR (1024) GENERATED ALWAYS AS (json ->> '$.entityFQN') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
UPDATE webhook_entity
|
||||
SET json = JSON_INSERT(json, '$.webhookType', 'generic');
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
ADD webhookType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.webhookType') NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS entity_extension_time_series (
|
||||
entityFQN VARCHAR(1024) NOT NULL, -- Entity FQN, we can refer to tables and columns
|
||||
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
|
||||
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
|
||||
json JSON NOT NULL,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD announcementStart BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.announcement.startTime'),
|
||||
ADD announcementEnd BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.announcement.endTime');
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.databaseSchema','$.connection.config.oracleServiceName')
|
||||
WHERE serviceType = 'Oracle';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.hostPort')
|
||||
WHERE serviceType = 'Athena';
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.username', '$.connection.config.password')
|
||||
WHERE serviceType in ('Databricks');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS openmetadata_settings (
|
||||
id MEDIUMINT NOT NULL AUTO_INCREMENT,
|
||||
configType VARCHAR(36) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id, configType),
|
||||
UNIQUE(configType)
|
||||
);
|
||||
|
||||
DELETE FROM entity_extension
|
||||
WHERE jsonSchema IN ('tableProfile', 'columnTest', 'tableTest');
|
||||
|
||||
DELETE FROM ingestion_pipeline_entity
|
||||
WHERE LOWER(JSON_EXTRACT(json, '$.pipelineType') = 'profiler');
|
||||
|
||||
DELETE FROM role_entity;
|
||||
DELETE FROM policy_entity;
|
||||
DELETE FROM field_relationship WHERE fromType IN ('role', 'policy') OR toType IN ('role', 'policy');
|
||||
DELETE FROM entity_relationship WHERE fromEntity IN ('role', 'policy') OR toEntity IN ('role', 'policy');
|
||||
ALTER TABLE role_entity DROP COLUMN defaultRole;
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
DELETE FROM entity_relationship
|
||||
WHERE toEntity = 'ingestionPipeline'
|
||||
AND toId NOT IN (
|
||||
SELECT DISTINCT id
|
||||
FROM ingestion_pipeline_entity
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_tokens (
|
||||
token VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.token') STORED NOT NULL,
|
||||
userId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.userId') STORED NOT NULL,
|
||||
tokenType VARCHAR(50) GENERATED ALWAYS AS (json ->> '$.tokenType') STORED NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
expiryDate BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.expiryDate') NOT NULL,
|
||||
PRIMARY KEY (token)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.metastoreHostPort'),
|
||||
'$.connection.config.metastoreConnection',
|
||||
JSON_OBJECT('metastoreHostPort', JSON_EXTRACT(json, '$.connection.config.metastoreHostPort'))
|
||||
)
|
||||
where serviceType = 'DeltaLake'
|
||||
and JSON_EXTRACT(json, '$.connection.config.metastoreHostPort') is not null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.metastoreFilePath'),
|
||||
'$.connection.config.metastoreConnection',
|
||||
JSON_OBJECT('metastoreFilePath', JSON_EXTRACT(json, '$.connection.config.metastoreFilePath'))
|
||||
)
|
||||
where serviceType = 'DeltaLake'
|
||||
and JSON_EXTRACT(json, '$.connection.config.metastoreFilePath') is not null;
|
||||
|
||||
|
||||
ALTER TABLE test_definition
|
||||
ADD COLUMN supported_data_types JSON GENERATED ALWAYS AS (json -> '$.supportedDataTypes');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableColumnCountToEqual'
|
||||
)
|
||||
WHERE BINARY name = 'TableColumnCountToEqual';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableColumnCountToEqual'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableColumnCountToEqual"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableColumnToMatchSet'
|
||||
)
|
||||
WHERE BINARY name = 'TableColumnToMatchSet';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableColumnToMatchSet'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableColumnToMatchSet"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableColumnNameToExist'
|
||||
)
|
||||
WHERE BINARY name = 'TableColumnNameToExist';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableColumnNameToExist'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableColumnNameToExist"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableRowCountToBeBetween'
|
||||
)
|
||||
WHERE BINARY name = 'TableRowCountToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableRowCountToBeBetween'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableRowCountToBeBetween"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableColumnCountToBeBetween'
|
||||
)
|
||||
WHERE BINARY name = 'TableColumnCountToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableColumnCountToBeBetween'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableColumnCountToBeBetween"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'columnValuesToBeInSet'
|
||||
)
|
||||
WHERE BINARY name = 'ColumnValuesToBeInSet';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'columnValuesToBeInSet'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"ColumnValuesToBeInSet"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
'tableRowCountToEqual'
|
||||
)
|
||||
WHERE BINARY name = 'TableRowCountToEqual';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
'tableRowCountToEqual'
|
||||
)
|
||||
WHERE BINARY JSON_CONTAINS(json, '"TableRowCountToEqual"', '$.fullyQualifiedName');
|
||||
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValueMaxToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'ARRAY', 'SET')
|
||||
)
|
||||
WHERE name = 'columnValueMeanToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValueMedianToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValueMinToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR', 'ARRAY')
|
||||
)
|
||||
WHERE name = 'columnValueLengthsToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
|
||||
)
|
||||
WHERE name = 'columnValuesMissingCount'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValuesSumToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValueStdDevToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeInSet'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeNotInSet'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeNotNull'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeUnique'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
|
||||
)
|
||||
WHERE name = 'columnValuesToMatchRegex'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
json,
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
|
||||
)
|
||||
WHERE name = 'columnValuesToNotMatchRegex'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.dbConnection')
|
||||
WHERE serviceType = 'Dagster';
|
||||
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET JSON = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.type','$.serviceType'),
|
||||
'$.connection.config.type','GluePipeline',
|
||||
'$.serviceType','GluePipeline'
|
||||
)
|
||||
where serviceType='Glue';
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
--
|
||||
-- Upgrade changes for 0.13
|
||||
--
|
||||
ALTER TABLE `entity_extension_time_series` modify entityFQN varchar(768);
|
||||
ALTER TABLE `entity_extension_time_series` ADD INDEX `entity_fqn_index` (`entityFQN`);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS web_analytic_event (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE(name),
|
||||
INDEX name_index (name)
|
||||
);
|
||||
|
||||
UPDATE bot_entity
|
||||
SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_insight_chart (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.dataIndexType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE(name),
|
||||
INDEX name_index (name)
|
||||
);
|
||||
|
||||
UPDATE role_entity
|
||||
SET json = JSON_INSERT(json, '$.provider', 'system')
|
||||
WHERE name in ('DataConsumer', 'DataSteward');
|
||||
|
||||
UPDATE policy_entity
|
||||
SET json = JSON_INSERT(json, '$.provider', 'system')
|
||||
WHERE fullyQualifiedName in (json, 'DataConsumerPolicy', 'DataStewardPolicy', 'OrganizationPolicy', 'TeamOnlyPolicy');
|
||||
|
||||
UPDATE tag_category
|
||||
SET json = JSON_INSERT(json, '$.provider', 'system')
|
||||
WHERE name in ('PersonalData', 'PII', 'Tier');
|
||||
|
||||
UPDATE tag
|
||||
SET json = JSON_INSERT(json, '$.provider', 'system')
|
||||
WHERE fullyQualifiedName in ('PersonalData.Personal', 'PersonalData.SpecialCategory',
|
||||
'PII.None', 'PII.NonSensitive', 'PII.Sensitive',
|
||||
'Tier.Tier1', 'Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5');
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_INSERT(json ,'$.connection.config.configSource.hostPort', '$.connection.config.hostPort')
|
||||
WHERE serviceType = 'Dagster';
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_REMOVE(json ,'$.connection.config.hostPort', '$.connection.config.numberOfStatus')
|
||||
WHERE serviceType = 'Dagster';
|
||||
|
||||
-- Remove categoryType
|
||||
UPDATE tag_category
|
||||
SET json = JSON_REMOVE(json ,'$.categoryType');
|
||||
|
||||
-- Set mutuallyExclusive flag
|
||||
UPDATE tag_category
|
||||
SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'false');
|
||||
|
||||
UPDATE tag_category
|
||||
SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'true')
|
||||
WHERE name in ('PersonalData', 'PII', 'Tier');
|
||||
|
||||
UPDATE tag
|
||||
SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'false');
|
||||
|
||||
-- Merge mutually exclusive table when multiple tag labels are used
|
||||
DELETE t1 FROM tag_usage t1
|
||||
INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
|
||||
WHERE (t2.tagFQN = 'Tier.Tier1' AND t1.tagFQN in ('Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')) OR
|
||||
(t2.tagFQN = 'Tier.Tier2' AND t1.tagFQN in ('Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')) OR
|
||||
(t2.tagFQN = 'Tier.Tier3' AND t1.tagFQN in ('Tier.Tier4', 'Tier.Tier5')) OR
|
||||
(t2.tagFQN = 'Tier.Tier4' AND t1.tagFQN in ('Tier.Tier5'));
|
||||
|
||||
DELETE t1 FROM tag_usage t1
|
||||
INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
|
||||
WHERE (t2.tagFQN = 'PII.Sensitive' AND t1.tagFQN in ('PII.NonSensitive', 'PII.None')) OR
|
||||
(t2.tagFQN = 'PII.NonSensitive' AND t1.tagFQN in ('PII.None'));
|
||||
|
||||
DELETE t1 FROM tag_usage t1
|
||||
INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
|
||||
WHERE (t2.tagFQN = 'PersonalData.Personal' AND t1.tagFQN in ('PersonalData.SpecialCategory'));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kpi_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
-- We are starting to store the current deployed flag. Let's mark it as false by default
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json ,'$.deployed');
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_INSERT(json ,'$.deployed', 'true');
|
||||
|
||||
-- We removed the supportsMetadataExtraction field in the `OpenMetadataConnection` object being used in IngestionPipelines
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json ,'$.openMetadataServerConnection.supportsMetadataExtraction');
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
-- Remove markDeletedTablesFromFilterOnly
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json ,'$.sourceConfig.config.markDeletedTablesFromFilterOnly');
|
||||
|
||||
UPDATE data_insight_chart
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.dimensions'),
|
||||
'$.dimensions',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT('name', 'entityFqn', 'chartDataType', 'STRING'),
|
||||
JSON_OBJECT('name', 'owner', 'chartDataType', 'STRING'),
|
||||
JSON_OBJECT('name', 'entityType', 'chartDataType', 'STRING'),
|
||||
JSON_OBJECT('name', 'entityHref', 'chartDataType', 'STRING')
|
||||
)
|
||||
)
|
||||
WHERE name = 'mostViewedEntities';
|
||||
|
||||
DROP TABLE webhook_entity;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_action_def (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
alertActionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.alertActionType') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.databaseSchema'),
|
||||
'$.connection.config.database',
|
||||
JSON_EXTRACT(json, '$.connection.config.databaseSchema')
|
||||
) where serviceType in ('Db2');
|
||||
|
||||
DELETE from openmetadata_settings where configType = 'activityFeedFilterSetting';
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json ,'$.sourceConfig.config.dbtConfigSource');
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.host')),'$.connection.config.token',JSON_EXTRACT(json, '$.connection.config.configSource.token'))
|
||||
WHERE serviceType = 'Dagster' AND json -> '$.connection.config.configSource.host' IS NOT NULL;
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.hostPort')), '$.connection.config.token','')
|
||||
WHERE serviceType = 'Dagster' AND json -> '$.connection.config.configSource.hostPort' IS NOT NULL;
|
||||
|
||||
UPDATE topic_entity
|
||||
SET json = JSON_INSERT(JSON_REMOVE(json, '$.schemaType'), '$.messageSchema', JSON_OBJECT('schemaType', JSON_EXTRACT(json, '$.schemaType')))
|
||||
WHERE json -> '$.schemaType' IS NOT NULL;
|
||||
|
||||
UPDATE topic_entity
|
||||
SET json = JSON_INSERT(JSON_REMOVE(json, '$.schemaText'), '$.messageSchema.schemaText', JSON_EXTRACT(json, '$.schemaText'))
|
||||
WHERE json -> '$.schemaText' IS NOT NULL;
|
||||
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
RENAME TABLE tag_category TO classification;
|
||||
|
||||
-- Rename tagCategoryName in BigQuery for classificationName
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.tagCategoryName'),
|
||||
'$.connection.config.classificationName',
|
||||
JSON_EXTRACT(json, '$.connection.config.tagCategoryName')
|
||||
) where serviceType in ('BigQuery');
|
||||
|
||||
-- Delete supportsUsageExtraction from vertica
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.supportsUsageExtraction')
|
||||
WHERE serviceType = 'Vertica';
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json ,'$.sourceConfig.config.dbtConfigSource.dbtUpdateDescriptions')
|
||||
WHERE json -> '$.sourceConfig.config.type' = 'DBT';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.supportedDataTypes'),
|
||||
'$.supportedDataTypes',
|
||||
JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'TIMESTAMP', 'TIMESTAMPZ','DATETIME', 'DATE')
|
||||
)
|
||||
WHERE name = 'columnValuesToBeBetween';
|
||||
|
||||
UPDATE pipeline_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.name'),
|
||||
'$.name',
|
||||
REPLACE(JSON_UNQUOTE(JSON_EXTRACT(json, '$.name')),':','')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.serviceType') = 'Dagster';
|
||||
|
||||
UPDATE pipeline_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.fullyQualifiedName'),
|
||||
'$.fullyQualifiedName',
|
||||
REPLACE(JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')),':','')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.serviceType') = 'Dagster';
|
||||
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json,'$.connection.config.username','$.connection.config.password','$.connection.config.provider'),
|
||||
'$.connection.config.connection',
|
||||
JSON_OBJECT(
|
||||
'username',JSON_EXTRACT(json,'$.connection.config.username'),
|
||||
'password',JSON_EXTRACT(json,'$.connection.config.password'),
|
||||
'provider',JSON_EXTRACT(json,'$.connection.config.provider')
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'Superset';
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
-- Unique constraint for user email address
|
||||
ALTER TABLE user_entity
|
||||
ADD UNIQUE (email);
|
||||
|
||||
|
||||
-- Remove classificationName in BigQuery
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.classificationName') where serviceType in ('BigQuery');
|
||||
|
||||
-- migrate ingestAllDatabases in postgres
|
||||
UPDATE dbservice_entity de2
|
||||
SET json = JSON_REPLACE(
|
||||
JSON_INSERT(json,
|
||||
'$.connection.config.database',
|
||||
(select JSON_EXTRACT(json, '$.name')
|
||||
from database_entity de
|
||||
where id = (select er.toId
|
||||
from entity_relationship er
|
||||
where er.fromId = de2.id
|
||||
and er.toEntity = 'database'
|
||||
LIMIT 1
|
||||
))
|
||||
), '$.connection.config.ingestAllDatabases',
|
||||
true
|
||||
)
|
||||
where de2.serviceType = 'Postgres'
|
||||
and JSON_EXTRACT(json, '$.connection.config.database') is NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS storage_container_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_connection_definition (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS automations_workflow (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.workflowType') STORED NOT NULL,
|
||||
status VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.status') STORED,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
-- Do not store OM server connection, we'll set it dynamically on the resource
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json, '$.openMetadataServerConnection');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS query_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE(name),
|
||||
INDEX name_index (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS temp_query_migration (
|
||||
tableId VARCHAR(36)NOT NULL,
|
||||
queryId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
|
||||
queryName VARCHAR(255) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
json JSON NOT NULL
|
||||
);
|
||||
|
||||
|
||||
INSERT INTO temp_query_migration(tableId,json)
|
||||
SELECT id,JSON_OBJECT('id',UUID(),'query',query,'users',users,'checksum',checksum,'duration',duration,'name',checksum,'updatedAt',UNIX_TIMESTAMP(NOW()),'updatedBy','admin','deleted',false) as json from entity_extension d, json_table(d.json, '$[*]' columns (query varchar(200) path '$.query',users json path '$.users',checksum varchar(200) path '$.checksum',name varchar(255) path '$.checksum', duration double path '$.duration',queryDate varchar(200) path '$.queryDate')) AS j WHERE extension = 'table.tableQueries';
|
||||
|
||||
INSERT INTO query_entity (json)
|
||||
SELECT t.json from temp_query_migration t
|
||||
ON DUPLICATE KEY UPDATE json = VALUES(json);
|
||||
|
||||
INSERT INTO entity_relationship(fromId,toId,fromEntity,toEntity,relation)
|
||||
SELECT tmq.tableId, (select qe.id from query_entity qe where qe.name = tmq.queryName) ,'table','query',5 FROM temp_query_migration tmq;
|
||||
|
||||
DELETE FROM entity_extension WHERE id IN
|
||||
(SELECT DISTINCT tableId FROM temp_query_migration) AND extension = 'table.tableQueries';
|
||||
|
||||
DROP Table temp_query_migration;
|
||||
|
||||
-- remove the audience if it was wrongfully sent from the UI after editing the OM service
|
||||
UPDATE metadata_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.securityConfig.audience')
|
||||
WHERE name = 'OpenMetadata' AND JSON_EXTRACT(json, '$.connection.config.authProvider') != 'google';
|
||||
|
||||
ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.expiryDate');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS event_subscription_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
|
||||
);
|
||||
|
||||
drop table if exists alert_action_def;
|
||||
drop table if exists alert_entity;
|
||||
DELETE from entity_relationship where fromEntity = 'alert' and toEntity = 'alertAction';
|
||||
|
||||
-- create data model table
|
||||
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.database'),
|
||||
'$.connection.config.databaseName', JSON_EXTRACT(json, '$.connection.config.database')
|
||||
)
|
||||
where serviceType = 'Druid'
|
||||
and JSON_EXTRACT(json, '$.connection.config.database') is not null;
|
||||
|
||||
-- We were using the same jsonSchema for Pipeline Services and Ingestion Pipeline status
|
||||
-- Also, we relied on the extension to store the run id
|
||||
UPDATE entity_extension_time_series
|
||||
SET jsonSchema = 'ingestionPipelineStatus', extension = 'ingestionPipeline.pipelineStatus'
|
||||
WHERE jsonSchema = 'pipelineStatus' AND extension <> 'pipeline.PipelineStatus';
|
||||
|
||||
-- We are refactoring the storage service with containers. We'll remove the locations
|
||||
DROP TABLE location_entity;
|
||||
DELETE FROM entity_relationship WHERE fromEntity='location' OR toEntity='location';
|
||||
TRUNCATE TABLE storage_service_entity;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.storageServiceName')
|
||||
WHERE serviceType = 'Glue';
|
||||
|
||||
UPDATE chart_entity
|
||||
SET json = JSON_REMOVE(json, '$.tables');
|
||||
|
||||
-- Updating the tableau authentication fields
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json,'$.connection.config.username','$.connection.config.password'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT(
|
||||
'username',JSON_EXTRACT(json,'$.connection.config.username'),
|
||||
'password',JSON_EXTRACT(json,'$.connection.config.password')
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'Tableau'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.username') is not null
|
||||
AND JSON_EXTRACT(json, '$.connection.config.password') is not null;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json,'$.connection.config.personalAccessTokenName','$.connection.config.personalAccessTokenSecret'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT(
|
||||
'personalAccessTokenName',JSON_EXTRACT(json,'$.connection.config.personalAccessTokenName'),
|
||||
'personalAccessTokenSecret',JSON_EXTRACT(json,'$.connection.config.personalAccessTokenSecret')
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'Tableau'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.personalAccessTokenName') is not null
|
||||
AND JSON_EXTRACT(json, '$.connection.config.personalAccessTokenSecret') is not null;
|
||||
|
||||
-- Removed property from metadataService.json
|
||||
UPDATE metadata_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.allowServiceCreation')
|
||||
WHERE serviceType in ('Amundsen', 'Atlas', 'MetadataES', 'OpenMetadata');
|
||||
|
||||
UPDATE metadata_service_entity
|
||||
SET json = JSON_INSERT(json, '$.provider', 'system')
|
||||
WHERE name = 'OpenMetadata';
|
||||
|
||||
-- Fix Glue sample data endpoint URL to be a correct URI
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REPLACE(json, '$.connection.config.awsConfig.endPointURL', 'https://glue.region_name.amazonaws.com/')
|
||||
WHERE serviceType = 'Glue'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.awsConfig.endPointURL') = 'https://glue.<region_name>.amazonaws.com/';
|
||||
|
||||
-- Delete connectionOptions from superset
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions')
|
||||
WHERE serviceType = 'Superset';
|
||||
|
||||
-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.partitionQueryDuration', '$.connection.config.partitionQuery', '$.connection.config.partitionField')
|
||||
WHERE serviceType = 'BigQuery';
|
||||
|
||||
-- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.scheme', '$.connection.config.hostPort', '$.connection.config.supportsProfiler', '$.connection.config.supportsQueryComment')
|
||||
WHERE serviceType = 'Salesforce';
|
||||
|
||||
-- Delete supportsProfiler from DynamoDB
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler')
|
||||
WHERE serviceType = 'DynamoDB';
|
||||
|
||||
-- Update TagLabels source from 'Tag' to 'Classification' after #10486
|
||||
UPDATE table_entity SET json = REGEXP_REPLACE(json, '\"source\"\\s*:\\s*\"Tag\"', '\"source\": \"Classification\"');
|
||||
UPDATE ml_model_entity SET json = REGEXP_REPLACE(json, '\"source\"\\s*:\\s*\"Tag\"', '\"source\": \"Classification\"');
|
||||
|
||||
|
||||
-- Delete supportsProfiler from Mssql
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.uriString')
|
||||
WHERE serviceType = 'Mssql';
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
-- Updating the value of SASL Mechanism for Kafka and Redpanda connections
|
||||
UPDATE messaging_service_entity
|
||||
SET json = JSON_REPLACE(json, '$.connection.config.saslMechanism', 'PLAIN')
|
||||
WHERE (serviceType = 'Kafka' OR serviceType = 'Redpanda')
|
||||
AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') IS NOT NULL
|
||||
AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') NOT IN ('GSSAPI', 'PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512', 'OAUTHBEARER');
|
||||
|
||||
-- Remove the Subscriptions
|
||||
DELETE FROM event_subscription_entity;
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
|
||||
-- Remove the ibmi scheme from Db2 replace it with db2+ibm_db
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_SET(json, '$.connection.config.scheme', 'db2+ibm_db')
|
||||
WHERE serviceType = 'Db2';
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
-- Update the tableau data model enum
|
||||
UPDATE dashboard_data_model_entity
|
||||
SET json = JSON_SET(json, '$.dataModelType', 'TableauDataModel')
|
||||
WHERE JSON_EXTRACT(json, '$.dataModelType') = 'TableauSheet';
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- use FQN instead of name for Test Connection Definition
|
||||
ALTER TABLE test_connection_definition
|
||||
ADD fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
DROP COLUMN name;
|
||||
|
||||
-- Since we are not deleting the test connection defs anymore, clean it up
|
||||
TRUNCATE TABLE test_connection_definition;
|
||||
+363
@@ -0,0 +1,363 @@
|
||||
-- we are not using the secretsManagerCredentials
|
||||
UPDATE metadata_service_entity
|
||||
SET json = JSON_REMOVE(json, '$.openMetadataServerConnection.secretsManagerCredentials')
|
||||
where name = 'OpenMetadata';
|
||||
|
||||
-- Rename githubCredentials to gitCredentials
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.githubCredentials'),
|
||||
'$.connection.config.gitCredentials',
|
||||
JSON_EXTRACT(json, '$.connection.config.githubCredentials')
|
||||
)
|
||||
WHERE serviceType = 'Looker'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.githubCredentials') IS NOT NULL;
|
||||
|
||||
|
||||
-- Rename gcsConfig in BigQuery to gcpConfig
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.credentials.gcsConfig'),
|
||||
'$.connection.config.credentials.gcpConfig',
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcsConfig')
|
||||
) where serviceType in ('BigQuery');
|
||||
|
||||
-- Rename gcsConfig in Datalake to gcpConfig
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.configSource.securityConfig.gcsConfig'),
|
||||
'$.connection.config.configSource.securityConfig.gcpConfig',
|
||||
JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcsConfig')
|
||||
) where serviceType in ('Datalake')
|
||||
AND JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcsConfig') IS NOT NULL;
|
||||
|
||||
|
||||
-- Rename gcsConfig in dbt to gcpConfig
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig'),
|
||||
'$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig',
|
||||
JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig')
|
||||
)
|
||||
WHERE json -> '$.sourceConfig.config.type' = 'DBT'
|
||||
AND JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig') IS NOT NULL;
|
||||
|
||||
-- Rename chartUrl in chart_entity to sourceUrl
|
||||
UPDATE chart_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.chartUrl'),
|
||||
'$.sourceUrl',
|
||||
JSON_EXTRACT(json, '$.chartUrl')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.chartUrl') IS NOT NULL;
|
||||
|
||||
-- Rename dashboardUrl in dashboard_entity to sourceUrl
|
||||
UPDATE dashboard_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.dashboardUrl'),
|
||||
'$.sourceUrl',
|
||||
JSON_EXTRACT(json, '$.dashboardUrl')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.dashboardUrl') IS NOT NULL;
|
||||
|
||||
-- Rename pipelineUrl in pipeline_entity to sourceUrl
|
||||
UPDATE pipeline_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.pipelineUrl'),
|
||||
'$.sourceUrl',
|
||||
JSON_EXTRACT(json, '$.pipelineUrl')
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.pipelineUrl') IS NOT NULL;
|
||||
|
||||
|
||||
-- Rename taskUrl in pipeline_entity to sourceUrl
|
||||
UPDATE pipeline_entity AS pe
|
||||
JOIN (
|
||||
SELECT id, JSON_ARRAYAGG(JSON_OBJECT(
|
||||
'name', t.name,
|
||||
'sourceUrl', t.sourceUrl,
|
||||
'taskType', t.taskType,
|
||||
'description', t.description,
|
||||
'displayName', t.displayName,
|
||||
'fullyQualifiedName', t.fullyQualifiedName,
|
||||
'downstreamTasks', t.downstreamTasks,
|
||||
'tags', t.tags,
|
||||
'endDate', t.endDate,
|
||||
'startDate', t.startDate,
|
||||
'taskSQL', t.taskSQL
|
||||
)) AS updated_json
|
||||
FROM pipeline_entity,
|
||||
JSON_TABLE(
|
||||
json,
|
||||
'$.tasks[*]' COLUMNS (
|
||||
name VARCHAR(256) PATH '$.name',
|
||||
sourceUrl VARCHAR(256) PATH '$.taskUrl',
|
||||
taskType VARCHAR(256) PATH '$.taskType',
|
||||
description TEXT PATH '$.description',
|
||||
displayName VARCHAR(256) PATH '$.displayName',
|
||||
fullyQualifiedName VARCHAR(256) PATH '$.fullyQualifiedName',
|
||||
downstreamTasks JSON PATH '$.downstreamTasks',
|
||||
tags JSON PATH '$.tags',
|
||||
endDate VARCHAR(256) PATH '$.endDate',
|
||||
startDate VARCHAR(256) PATH '$.startDate',
|
||||
taskSQL TEXT PATH '$.taskSQL'
|
||||
)
|
||||
) AS t
|
||||
GROUP BY id
|
||||
) AS updated_data
|
||||
ON pe.id = updated_data.id
|
||||
SET pe.json = JSON_INSERT(
|
||||
JSON_REMOVE(pe.json, '$.tasks'),
|
||||
'$.tasks',
|
||||
updated_data.updated_json
|
||||
);
|
||||
|
||||
-- Modify migrations for service connection of postgres and mysql to move password under authType
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.password'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.authType.password',
|
||||
JSON_EXTRACT(json, '$.connection.config.password'))
|
||||
where serviceType in ('Postgres', 'Mysql');
|
||||
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
|
||||
-- Remove sourceUrl in pipeline_entity from DatabricksPipeline & Fivetran
|
||||
UPDATE pipeline_entity
|
||||
SET json = JSON_REMOVE(json, '$.sourceUrl')
|
||||
WHERE JSON_EXTRACT(json, '$.serviceType') in ('DatabricksPipeline','Fivetran');
|
||||
|
||||
-- Remove sourceUrl in dashboard_entity from Mode
|
||||
UPDATE dashboard_entity
|
||||
SET json = JSON_REMOVE(json, '$.sourceUrl')
|
||||
WHERE JSON_EXTRACT(json, '$.serviceType') in ('Mode');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
|
||||
installed_rank SERIAL,
|
||||
version VARCHAR(256) PRIMARY KEY,
|
||||
migrationFileName VARCHAR(256) NOT NULL,
|
||||
checksum VARCHAR(256) NOT NULL,
|
||||
installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
|
||||
version VARCHAR(256) NOT NULL,
|
||||
sqlStatement VARCHAR(10000) NOT NULL,
|
||||
checksum VARCHAR(256) PRIMARY KEY,
|
||||
executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Update test definition parameterValues
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minValueForMeanInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected mean value for the column to be greater or equal than',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxValueForMeanInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected mean value for the column to be lower or equal than',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'columnValueMeanToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minValueForMedianInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected median value for the column to be greater or equal than',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxValueForMedianInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected median value for the column to be lower or equal than',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'columnValueMedianToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minValueForStdDevInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected std. dev value for the column to be greater or equal than',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxValueForStdDevInCol',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected std. dev value for the column to be lower or equal than',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'columnValueStdDevToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minLength',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxLength',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'columnValueLengthsToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minValue',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxValue',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'columnValuesToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'columnNames',
|
||||
'dataType', 'STRING',
|
||||
'required', true,
|
||||
'description', 'Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string',
|
||||
'displayName', 'Column Names'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'ordered',
|
||||
'dataType', 'BOOLEAN',
|
||||
'required', false,
|
||||
'description', 'Whether or not to considered the order of the list when performing the match check',
|
||||
'displayName', 'Ordered'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'tableColumnToMatchSet';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'minValue',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum',
|
||||
'displayName', 'Min'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'maxValue',
|
||||
'dataType', 'INT',
|
||||
'required', false,
|
||||
'description', 'Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum',
|
||||
'displayName', 'Max'
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'tableRowCountToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.parameterDefinition'),
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'sqlExpression',
|
||||
'displayName', 'SQL Expression',
|
||||
'description', 'SQL expression to run against the table',
|
||||
'dataType', 'STRING',
|
||||
'required', 'true'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'strategy',
|
||||
'displayName', 'Strategy',
|
||||
'description', 'Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to ROWS)',
|
||||
'dataType', 'ARRAY',
|
||||
'optionValues', JSON_ARRAY(
|
||||
'ROWS',
|
||||
'COUNT'
|
||||
),
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'threshold',
|
||||
'displayName', 'Threshold',
|
||||
'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
|
||||
'dataType', 'NUMBER',
|
||||
'required', false
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE name = 'tableCustomSQLQuery';
|
||||
|
||||
-- Modify migrations for service connection of airflow to move password under authType if
|
||||
-- Connection Type as Mysql or Postgres
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.connection.password'),
|
||||
'$.connection.config.connection.authType',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.connection.authType.password',
|
||||
JSON_EXTRACT(json, '$.connection.config.connection.password'))
|
||||
where serviceType = 'Airflow'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.connection.type') in ('Postgres', 'Mysql')
|
||||
AND JSON_EXTRACT(json, '$.connection.config.connection.password') IS NOT NULL;
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
-- Rename includeTempTables with includeTransTables
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(
|
||||
JSON_SET(
|
||||
json,
|
||||
'$.connection.config.includeTransientTables',
|
||||
JSON_EXTRACT(json, '$.connection.config.includeTempTables')
|
||||
),
|
||||
'$.connection.config.includeTempTables'
|
||||
)
|
||||
WHERE serviceType in ('Snowflake') AND JSON_EXTRACT(json, '$.connection.config.includeTempTables') IS NOT NULL;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REPLACE(json, '$.connection.config.scheme', 'hive')
|
||||
WHERE JSON_EXTRACT(json, '$.connection.config.scheme') IN ('impala', 'impala4')
|
||||
AND serviceType = 'Hive';
|
||||
|
||||
-- remove the dataModel references from Data Models
|
||||
UPDATE dashboard_data_model_entity
|
||||
SET json = JSON_REMOVE(json, '$.dataModels');
|
||||
|
||||
-- migrate ingestAllDatabases in mssql
|
||||
UPDATE dbservice_entity de2
|
||||
SET json = JSON_REPLACE(
|
||||
JSON_INSERT(json,
|
||||
'$.connection.config.database',
|
||||
(select JSON_EXTRACT(json, '$.name')
|
||||
from database_entity de
|
||||
where id = (select er.toId
|
||||
from entity_relationship er
|
||||
where er.fromId = de2.id
|
||||
and er.toEntity = 'database'
|
||||
LIMIT 1
|
||||
))
|
||||
), '$.connection.config.ingestAllDatabases',
|
||||
true
|
||||
)
|
||||
where de2.serviceType = 'Mssql'
|
||||
and JSON_EXTRACT(json, '$.connection.config.database') is NULL;
|
||||
|
||||
-- remove keyfile from clickhouse
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.keyfile')
|
||||
WHERE serviceType = 'Clickhouse';
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
-- This column is already created in v000__create_server_change_log.sql
|
||||
-- ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics JSON;
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
-- Create tables for tracking server migrations
|
||||
-- This migration runs before all other migrations to ensure migration tracking tables exist
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
|
||||
installed_rank SERIAL,
|
||||
version VARCHAR(256) PRIMARY KEY,
|
||||
migrationFileName VARCHAR(256) NOT NULL,
|
||||
checksum VARCHAR(256) NOT NULL,
|
||||
installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
metrics JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
|
||||
version VARCHAR(256) NOT NULL,
|
||||
sqlStatement VARCHAR(10000) NOT NULL,
|
||||
checksum VARCHAR(256) PRIMARY KEY,
|
||||
executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
CREATE OR REPLACE FUNCTION to_tz_timestamp(text) RETURNS TIMESTAMP WITH TIME ZONE AS
|
||||
$$
|
||||
select to_timestamp($1, '%Y-%m-%dT%T.%fZ')::timestamptz;
|
||||
$$
|
||||
LANGUAGE sql immutable;
|
||||
|
||||
--
|
||||
-- Table that captures all the relationships between entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_relationship (
|
||||
fromId VARCHAR(36) NOT NULL, -- ID of the from entity
|
||||
toId VARCHAR(36) NOT NULL, -- ID of the to entity
|
||||
fromEntity VARCHAR(256) NOT NULL, -- Type name of the from entity
|
||||
toEntity VARCHAR(256) NOT NULL, -- Type name of to entity
|
||||
relation SMALLINT NOT NULL,
|
||||
jsonSchema VARCHAR(256), -- Schema used for generating JSON
|
||||
json JSONB, -- JSON payload with additional information
|
||||
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
PRIMARY KEY (fromId, toId, relation)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS entity_relationship_edge_index ON entity_relationship(fromId, toId, relation);
|
||||
CREATE INDEX IF NOT EXISTS entity_relationship_from_index ON entity_relationship(fromId, relation);
|
||||
CREATE INDEX IF NOT EXISTS entity_relationship_to_index ON entity_relationship(toId, relation);
|
||||
|
||||
--
|
||||
-- Table that captures all the relationships between field of an entity to a field of another entity
|
||||
-- Example - table1.column1 (fromFQN of type table.columns.column) is joined with(relation)
|
||||
-- table2.column8 (toFQN of type table.columns.column)
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS field_relationship (
|
||||
fromFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
|
||||
toFQN VARCHAR(256) NOT NULL, -- Fully qualified name of entity or field
|
||||
fromType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
|
||||
toType VARCHAR(256) NOT NULL, -- Fully qualified type of entity or field
|
||||
relation SMALLINT NOT NULL,
|
||||
jsonSchema VARCHAR(256), -- Schema used for generating JSON
|
||||
json JSONB, -- JSON payload with additional information
|
||||
PRIMARY KEY (fromFQN, toFQN, relation)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS field_relationship_from_index ON field_relationship(fromFQN, relation);
|
||||
CREATE INDEX IF NOT EXISTS field_relationship_to_index ON field_relationship(toFQN, relation);
|
||||
|
||||
--
|
||||
-- Used for storing additional metadata for an entity
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_extension (
|
||||
id VARCHAR(36) NOT NULL, -- ID of the from entity
|
||||
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
|
||||
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id, extension)
|
||||
);
|
||||
|
||||
--
|
||||
-- Service entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS dbservice_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messaging_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dashboard_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pipeline_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS storage_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
--
|
||||
-- Data entities
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS database_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS database_schema_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt') :: bigint) STORED NOT NULL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted') :: boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS table_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metric_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dashboard_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pipeline_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS topic_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chart_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS location_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- Feed related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS thread_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityId') STORED NOT NULL,
|
||||
entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'about') STORED NOT NULL,
|
||||
assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> 'addressedTo') STORED,
|
||||
json JSONB NOT NULL,
|
||||
createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'threadTs')::bigint) STORED NOT NULL,
|
||||
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
resolved BOOLEAN GENERATED ALWAYS AS ((json ->> 'resolved')::boolean) STORED,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
--
|
||||
-- Policies related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS policy_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- Ingestion related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
timestamp BIGINT,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
--
|
||||
-- User, Team, and bots
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS team_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
email VARCHAR(256) GENERATED ALWAYS AS (json ->> 'email') STORED NOT NULL,
|
||||
deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> 'deactivated') STORED,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bot_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS role_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
defaultRole BOOLEAN GENERATED ALWAYS AS ((json ->> 'defaultRole')::boolean) STORED,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
--
|
||||
-- Usage table where usage for all the entities is captured
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS entity_usage (
|
||||
id VARCHAR(36) NOT NULL, -- Unique id of the entity
|
||||
entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
|
||||
usageDate DATE, -- date corresponding to the usage
|
||||
count1 INT, -- total daily count of use on usageDate
|
||||
count7 INT, -- rolling count of last 7 days going back from usageDate
|
||||
count30 INT, -- rolling count of last 30 days going back from usageDate
|
||||
percentile1 INT, -- percentile rank with in same entity for given usage date
|
||||
percentile7 INT, -- percentile rank with in same entity for last 7 days of usage
|
||||
percentile30 INT, -- percentile rank with in same entity for last 30 days of usage
|
||||
UNIQUE (usageDate, id)
|
||||
);
|
||||
|
||||
--
|
||||
-- Tag related tables
|
||||
--
|
||||
CREATE TABLE IF NOT EXISTS tag_category (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL, -- JSON stores category information and does not store children
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name) -- Unique tag category name
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL, -- JSON stores all tag attributes and does not store children
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag_usage (
|
||||
source SMALLINT NOT NULL, -- Source of the tag label
|
||||
tagFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the tag
|
||||
targetFQN VARCHAR(256) NOT NULL, -- Fully qualified name of the entity instance or corresponding field
|
||||
labelType SMALLINT NOT NULL, -- Type of tagging: manual, automated, propagated, derived
|
||||
state SMALLINT NOT NULL, -- State of tagging: suggested or confirmed
|
||||
UNIQUE (source, tagFQN, targetFQN)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS change_event (
|
||||
eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
|
||||
entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
|
||||
userName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'userName') STORED NOT NULL,
|
||||
eventTime BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
|
||||
json JSONB NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS change_event_event_type_index ON change_event(eventType);
|
||||
CREATE INDEX IF NOT EXISTS change_event_entity_type_index ON change_event(entityType);
|
||||
CREATE INDEX IF NOT EXISTS change_event_event_time_index ON change_event(eventTime);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS webhook_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
-- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS glossary_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS glossary_term_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
CREATE TABLE IF NOT EXISTS type_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
ADD status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED NOT NULL,
|
||||
DROP COLUMN deleted;
|
||||
|
||||
DROP INDEX entity_relationship_edge_index;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mlmodel_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
UPDATE thread_entity
|
||||
SET json = jsonb_set(json, '{type}', '"Conversation"', true);
|
||||
|
||||
update thread_entity
|
||||
SET json = jsonb_set(json, '{reactions}', '[]'::jsonb, true);
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD type VARCHAR(64) GENERATED ALWAYS AS (json ->> 'type') STORED NOT NULL,
|
||||
ADD taskId INT GENERATED ALWAYS AS ((json#>'{task,id}')::integer) STORED,
|
||||
ADD taskStatus VARCHAR(64) GENERATED ALWAYS AS (json#>>'{task,status}') STORED,
|
||||
ADD taskAssignees JSONB GENERATED ALWAYS AS (json#>'{task,assignees}') STORED,
|
||||
ADD CONSTRAINT task_id_constraint UNIQUE(taskId);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS thread_entity_type_index ON thread_entity(type);
|
||||
CREATE INDEX IF NOT EXISTS thread_entity_task_assignees_index ON thread_entity(taskAssignees);
|
||||
CREATE INDEX IF NOT EXISTS thread_entity_task_status_index ON thread_entity(taskStatus);
|
||||
CREATE INDEX IF NOT EXISTS thread_entity_created_by_index ON thread_entity(createdBy);
|
||||
CREATE INDEX IF NOT EXISTS thread_entity_updated_at_index ON thread_entity(updatedAt);
|
||||
|
||||
CREATE TABLE task_sequence (id SERIAL PRIMARY KEY, dummy varchar(1));
|
||||
INSERT INTO task_sequence (dummy) VALUES (0) RETURNING id;
|
||||
|
||||
DELETE from ingestion_pipeline_entity where 1=1;
|
||||
DELETE FROM pipeline_service_entity WHERE 1=1;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json, '{connection,config,databaseSchema}', json#>'{connection,config,database}')
|
||||
where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle')
|
||||
and json#>'{connection,config,database}' is not null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,database}'
|
||||
where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle');
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,database}' #- '{connection,config,username}' #- '{connection,config,projectId}' #- '{connection,config,enablePolicyTagImport}'
|
||||
WHERE serviceType = 'BigQuery';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,database}'
|
||||
WHERE serviceType in ('Athena','Databricks');
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,supportsProfiler}' #- '{connection,config,pipelineServiceName}'
|
||||
WHERE serviceType = 'Glue';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,dbServiceName}'
|
||||
WHERE serviceType in ('Metabase','Superset','Tableau');
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
DELETE from ingestion_pipeline_entity where 1=1;
|
||||
DELETE from entity_relationship where toEntity = 'ingestionPipeline';
|
||||
|
||||
-- 0.10 had empty FQNs for services, users and teams
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE messaging_service_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE storage_service_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE team_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
|
||||
UPDATE user_entity
|
||||
SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
|
||||
WHERE json#>'{fullyQualifiedName}' is NULL;
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
UPDATE team_entity
|
||||
SET json = JSONB_SET(json, '{teamType}', '"Group"', true);
|
||||
|
||||
ALTER TABLE team_entity
|
||||
ADD teamType VARCHAR(64) GENERATED ALWAYS AS (json ->> 'teamType') STORED NOT NULL;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,database}'
|
||||
where serviceType = 'DynamoDB';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,connectionOptions}'
|
||||
where serviceType = 'DeltaLake';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,supportsProfiler}'
|
||||
where serviceType = 'DeltaLake';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = jsonb_set(json, '{connection,config,clientId}', json#>'{connection,config,username}')
|
||||
WHERE serviceType = 'Looker'
|
||||
and json#>'{connection,config,username}' is not null;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = jsonb_set(json, '{connection,config,clientSecret}', json#>'{connection,config,password}')
|
||||
WHERE serviceType = 'Looker'
|
||||
and json#>'{connection,config,password}' is not null;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}' #- '{connection,config,env}'
|
||||
WHERE serviceType = 'Looker';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_definition (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_suite (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_case (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
entityFQN VARCHAR (712) GENERATED ALWAYS AS (json ->> 'entityFQN') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
UPDATE webhook_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{webhookType}', '"generic"', true);
|
||||
|
||||
ALTER TABLE webhook_entity
|
||||
ADD webhookType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'webhookType') STORED NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS entity_extension_time_series (
|
||||
entityFQN VARCHAR(1024) NOT NULL, -- Entity FQN, we can refer to tables and columns
|
||||
extension VARCHAR(256) NOT NULL, -- Extension name same as entity.fieldName
|
||||
jsonSchema VARCHAR(256) NOT NULL, -- Schema used for generating JSON
|
||||
json JSONB NOT NULL,
|
||||
timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
|
||||
);
|
||||
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD announcementStart BIGINT GENERATED ALWAYS AS ((json#>'{announcement,startTime}')::bigint) STORED,
|
||||
ADD announcementEnd BIGINT GENERATED ALWAYS AS ((json#>'{announcement,endTime}')::bigint) STORED;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,databaseSchema}' #- '{connection,config,oracleServiceName}'
|
||||
WHERE serviceType = 'Oracle';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,hostPort}'
|
||||
WHERE serviceType = 'Athena';
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}'
|
||||
WHERE serviceType in ('Databricks');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS openmetadata_settings (
|
||||
id SERIAL NOT NULL ,
|
||||
configType VARCHAR(36) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id, configType),
|
||||
UNIQUE(configType)
|
||||
);
|
||||
|
||||
DELETE FROM entity_extension
|
||||
WHERE jsonSchema IN ('tableProfile', 'columnTest', 'tableTest');
|
||||
|
||||
DELETE FROM ingestion_pipeline_entity
|
||||
WHERE json_extract_path_text("json", 'pipelineType') = 'profiler';
|
||||
|
||||
DELETE FROM role_entity;
|
||||
DELETE FROM policy_entity;
|
||||
DELETE FROM field_relationship WHERE fromType IN ('role', 'policy') OR toType IN ('role', 'policy');
|
||||
DELETE FROM entity_relationship WHERE fromEntity IN ('role', 'policy') OR toEntity IN ('role', 'policy');
|
||||
ALTER TABLE role_entity DROP COLUMN defaultRole;
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
DELETE FROM entity_relationship
|
||||
WHERE toEntity = 'ingestionPipeline'
|
||||
AND toId NOT IN (
|
||||
SELECT DISTINCT id
|
||||
FROM ingestion_pipeline_entity
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_tokens (
|
||||
token VARCHAR(36) GENERATED ALWAYS AS (json ->> 'token') STORED NOT NULL,
|
||||
userId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'userId') STORED NOT NULL,
|
||||
tokenType VARCHAR(50) GENERATED ALWAYS AS (json ->> 'tokenType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
expiryDate BIGINT GENERATED ALWAYS AS ((json ->> 'expiryDate')::bigint) STORED NOT NULL,
|
||||
PRIMARY KEY (token)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{connection,config,metastoreConnection}',
|
||||
jsonb_build_object('metastoreHostPort', json#>'{connection,config,metastoreHostPort}')
|
||||
)
|
||||
WHERE serviceType = 'DeltaLake'
|
||||
AND json#>'{connection,config,metastoreHostPort}' is not null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,metastoreHostPort}'
|
||||
WHERE serviceType = 'DeltaLake';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{connection,config,metastoreConnection}',
|
||||
jsonb_build_object('metastoreFilePath', json#>'{connection,config,metastoreFilePath}')
|
||||
)
|
||||
WHERE serviceType = 'DeltaLake'
|
||||
AND json#>'{connection,config,metastoreFilePath}' is not null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,metastoreFilePath}'
|
||||
WHERE serviceType = 'DeltaLake';
|
||||
|
||||
ALTER TABLE test_definition
|
||||
ADD supported_data_types JSONB GENERATED ALWAYS AS (json -> 'supportedDataTypes') STORED;
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableColumnCountToEqual"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableColumnCountToEqual';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableColumnCountToEqual"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableColumnCountToEqual';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableColumnToMatchSet"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableColumnToMatchSet';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableColumnToMatchSet"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableColumnToMatchSet';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableColumnNameToExist"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableColumnNameToExist';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableColumnNameToExist"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableColumnNameToExist';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableRowCountToBeBetween"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableRowCountToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableRowCountToBeBetween"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableRowCountToBeBetween';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableColumnCountToBeBetween"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableColumnCountToBeBetween';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableColumnCountToBeBetween"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableColumnCountToBeBetween';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"columnValuesToBeInSet"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'ColumnValuesToBeInSet';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"columnValuesToBeInSet"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'ColumnValuesToBeInSet';
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
'"tableRowCountToEqual"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'TableRowCountToEqual';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
'"tableRowCountToEqual"',
|
||||
false
|
||||
)
|
||||
WHERE json->>'fullyQualifiedName' = 'TableRowCountToEqual';
|
||||
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueMeanToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "ARRAY", "SET"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueMaxToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueMedianToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueMinToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueLengthsToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesMissingCount'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesSumToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValueStdDevToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeBetween'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeInSet'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeNotInSet'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeNotNull'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeUnique'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToMatchRegex'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToNotMatchRegex'
|
||||
AND supported_data_types IS NULL;
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,dbConnection}'
|
||||
WHERE serviceType = 'Dagster';
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
json,
|
||||
'{serviceType}',
|
||||
'"GluePipeline"',
|
||||
true
|
||||
),
|
||||
'{connection,config,type}',
|
||||
'"GluePipeline"',
|
||||
true
|
||||
)
|
||||
WHERE serviceType = 'Glue';
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
--
|
||||
-- Upgrade changes for 0.13
|
||||
--
|
||||
|
||||
|
||||
ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQN TYPE varchar(768);
|
||||
CREATE INDEX IF NOT EXISTS entity_extension_time_series_entity_fqn_index ON entity_extension_time_series(entityFQN);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS web_analytic_event (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
|
||||
|
||||
UPDATE bot_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
|
||||
|
||||
UPDATE bot_entity
|
||||
SET json = json::jsonb #- '{botType}';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_insight_chart (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'dataIndexType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
|
||||
|
||||
UPDATE role_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
|
||||
WHERE name in ('DataConsumer', 'DataSteward');
|
||||
|
||||
UPDATE policy_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
|
||||
WHERE fullyQualifiedName in ('DataConsumerPolicy', 'DataStewardPolicy', 'OrganizationPolicy', 'TeamOnlyPolicy');
|
||||
|
||||
UPDATE tag_category
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
|
||||
WHERE name in ('PersonalData', 'PII', 'Tier');
|
||||
|
||||
UPDATE tag
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
|
||||
WHERE fullyQualifiedName in ('PersonalData.Personal', 'PersonalData.SpecialCategory',
|
||||
'PII.None', 'PII.NonSensitive', 'PII.Sensitive',
|
||||
'Tier.Tier1', 'Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5');
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSONB_SET(json::jsonb,'{connection,config}',json::jsonb #>'{connection,config}' || jsonb_build_object('configSource',jsonb_build_object('hostPort',json #>'{connection,config,hostPort}')), true)
|
||||
where servicetype = 'Dagster';
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,hostPort}' #- '{connection,config,numberOfStatus}'
|
||||
where servicetype = 'Dagster';
|
||||
|
||||
-- Remove categoryType
|
||||
UPDATE tag_category
|
||||
SET json = json::jsonb #- '{categoryType}';
|
||||
|
||||
-- set mutuallyExclusive flag
|
||||
UPDATE tag_category
|
||||
SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
|
||||
|
||||
UPDATE tag_category
|
||||
SET json = jsonb_set(json, '{mutuallyExclusive}', 'true'::jsonb, true)
|
||||
WHERE name in ('PersonalData', 'PII', 'Tier');
|
||||
|
||||
UPDATE tag
|
||||
SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
|
||||
|
||||
-- Merge mutually exclusive table when multiple tag labels are used
|
||||
DELETE FROM tag_usage t1
|
||||
USING tag_usage t2
|
||||
WHERE
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'Tier.Tier1' AND
|
||||
t1.tagFQN in ('Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
|
||||
) OR
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'Tier.Tier2' AND
|
||||
t1.tagFQN in ('Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
|
||||
) OR
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'Tier.Tier3' AND
|
||||
t1.tagFQN in ('Tier.Tier4', 'Tier.Tier5')
|
||||
) OR
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'Tier.Tier4' AND
|
||||
t1.tagFQN in ('Tier.Tier5'));
|
||||
|
||||
DELETE FROM tag_usage t1
|
||||
USING tag_usage t2
|
||||
WHERE
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'PII.Sensitive' AND
|
||||
t1.tagFQN in ('PII.NonSensitive', 'PII.None')
|
||||
) OR
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'PII.NonSensitive' AND
|
||||
t1.tagFQN in ('PII.None'));
|
||||
|
||||
DELETE FROM tag_usage t1
|
||||
USING tag_usage t2
|
||||
WHERE
|
||||
(t1.targetFQN = t2.targetFQN AND
|
||||
t2.tagFQN = 'PersonalData.Personal' AND
|
||||
t1.tagFQN in ('PersonalData.SpecialCategory'));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kpi_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
-- We are starting to store the current deployed flag. Let's mark it as false by default
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{deployed}';
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = jsonb_set(json::jsonb, '{deployed}', 'true'::jsonb, true);
|
||||
|
||||
-- We removed the supportsMetadataExtraction field in the `OpenMetadataConnection` object being used in IngestionPipelines
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{openMetadataServerConnection,supportsMetadataExtraction}';
|
||||
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
-- Remove markDeletedTablesFromFilterOnly
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{sourceConfig,config,markDeletedTablesFromFilterOnly}';
|
||||
|
||||
UPDATE data_insight_chart
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{dimensions}',
|
||||
'[{"name":"entityFqn","chartDataType":"STRING"},{"name":"entityType","chartDataType":"STRING"},{"name":"owner","chartDataType":"STRING"},{"name":"entityHref","chartDataType":"STRING"}]'
|
||||
)
|
||||
WHERE name = 'mostViewedEntities';
|
||||
|
||||
DROP TABLE webhook_entity;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_action_def (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
alertActionType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'alertActionType') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json, '{connection,config,database}', json#>'{connection,config,databaseSchema}')
|
||||
where serviceType in ('Db2')
|
||||
and json#>'{connection,config,databaseSchema}' is not null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,databaseSchema}'
|
||||
where serviceType in ('Db2');
|
||||
|
||||
DELETE from openmetadata_settings where configType = 'activityFeedFilterSetting';
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource}';
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,token}', json#> '{connection,config,configSource,token}', true) ,'{connection,config,host}', json #> '{connection,config,configSource,host}' , true)
|
||||
WHERE serviceType = 'Dagster' and json #>'{connection,config,configSource,host}' is not null;
|
||||
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,host}', json#> '{connection,config,configSource,hostPort}', true)
|
||||
WHERE servicetype = 'Dagster' and json #>'{connection,config,configSource,hostPort}' is not null;
|
||||
|
||||
UPDATE topic_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{schemaText}', '{messageSchema}', jsonb_build_object('schemaText', json#>'{schemaText}'), true)
|
||||
WHERE json #> '{schemaText}' IS NOT NULL;
|
||||
|
||||
UPDATE topic_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{schemaType}', '{messageSchema,schemaType}', json#> '{schemaType}', true)
|
||||
WHERE json #> '{schemaType}' IS NOT NULL;
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
ALTER TABLE tag_category
|
||||
RENAME TO classification;
|
||||
|
||||
-- Rename tagCategoryName in BigQuery for classificationName
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json, '{connection,config,classificationName}', json#>'{connection,config,tagCategoryName}')
|
||||
where serviceType in ('BigQuery')
|
||||
and json#>'{connection,config,tagCategoryName}' is not null;
|
||||
|
||||
-- Delete supportsUsageExtraction from vertica
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,supportsUsageExtraction}'
|
||||
WHERE serviceType = 'Vertica';
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtUpdateDescriptions}'
|
||||
WHERE json#>>'{sourceConfig,config,type}' = 'DBT';
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "TIMESTAMP", "TIMESTAMPZ","DATETIME", "DATE"]',
|
||||
false
|
||||
)
|
||||
WHERE json->>'name' = 'columnValuesToBeBetween';
|
||||
|
||||
UPDATE pipeline_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{name}',
|
||||
to_jsonb(replace(json ->> 'name',':',''))
|
||||
)
|
||||
WHERE json ->> 'serviceType' = 'Dagster';
|
||||
|
||||
UPDATE pipeline_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{fullyQualifiedName}',
|
||||
to_jsonb(replace(json ->> 'fullyQualifiedName',':',''))
|
||||
)
|
||||
WHERE json ->> 'serviceType' = 'Dagster';
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSONB_SET(json::jsonb,
|
||||
'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}' #- '{provider}'||
|
||||
jsonb_build_object('connection',jsonb_build_object(
|
||||
'username',json #>'{connection,config,username}',
|
||||
'password',json #>'{connection,config,password}',
|
||||
'provider',json #>'{connection,config,provider}'
|
||||
)), true)
|
||||
where servicetype = 'Superset';
|
||||
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
-- Unique constraint for user email address
|
||||
ALTER TABLE user_entity
|
||||
ADD UNIQUE (email);
|
||||
|
||||
|
||||
-- Remove classificationName in BigQuery
|
||||
UPDATE dbservice_entity SET json = json #- '{connection,config,classificationName}' where serviceType in ('BigQuery');
|
||||
|
||||
-- migrate ingestAllDatabases in postgres
|
||||
UPDATE dbservice_entity de2
|
||||
SET json = JSONB_SET(
|
||||
json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'||
|
||||
jsonb_build_object('database',
|
||||
(SELECT json->>'name'
|
||||
FROM database_entity de
|
||||
WHERE id = (SELECT er.toId
|
||||
FROM entity_relationship er
|
||||
WHERE er.fromId = de2.id
|
||||
AND er.toEntity = 'database'
|
||||
LIMIT 1)
|
||||
)
|
||||
)),
|
||||
'{connection,config,ingestAllDatabases}',
|
||||
'true'::jsonb
|
||||
)
|
||||
WHERE de2.serviceType = 'Postgres'
|
||||
AND json->>'{connection,config,database}' IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS storage_container_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS test_connection_definition (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS automations_workflow (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'workflowType') STORED NOT NULL,
|
||||
status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
-- Do not store OM server connection, we'll set it dynamically on the resource
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{openMetadataServerConnection}';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS query_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS temp_query_migration (
|
||||
tableId VARCHAR(36) NOT NULL,
|
||||
queryId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
queryName VARCHAR(255) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
json JSONB NOT NULL
|
||||
);
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
|
||||
INSERT INTO temp_query_migration(tableId,json)
|
||||
SELECT id,json_build_object('id',gen_random_uuid(),'query',query,'users',users,'checksum',checksum,'duration',duration,'name',checksum,'updatedAt',
|
||||
floor(EXTRACT(EPOCH FROM NOW())),'updatedBy','admin','deleted',false) AS json FROM entity_extension AS ee , jsonb_to_recordset(ee.json) AS x (query varchar,users json,
|
||||
checksum varchar,name varchar, duration decimal,queryDate varchar)
|
||||
WHERE ee.extension = 'table.tableQueries';
|
||||
|
||||
INSERT INTO query_entity (json)
|
||||
SELECT value
|
||||
FROM (
|
||||
SELECT jsonb_object_agg(queryName, json) AS json_data FROM ( SELECT DISTINCT queryName, json FROM temp_query_migration) subquery
|
||||
) cte, jsonb_each(cte.json_data)
|
||||
ON CONFLICT (name) DO UPDATE SET json = EXCLUDED.json;
|
||||
|
||||
INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation)
|
||||
SELECT tmq.tableId, qe.id, 'table', 'query', 5
|
||||
FROM temp_query_migration tmq
|
||||
JOIN query_entity qe ON qe.name = tmq.queryName;
|
||||
|
||||
DELETE FROM entity_extension WHERE id in
|
||||
(SELECT DISTINCT tableId FROM temp_query_migration) AND extension = 'table.tableQueries';
|
||||
|
||||
DROP TABLE temp_query_migration;
|
||||
|
||||
-- remove the audience if it was wrongfully sent from the UI after editing the OM service
|
||||
UPDATE metadata_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,securityConfig,audience}'
|
||||
WHERE name = 'OpenMetadata'
|
||||
AND json#>'{connection,config,authProvider}' IS NOT NULL
|
||||
AND json -> 'connection' -> 'config' ->> 'authProvider' != 'google';
|
||||
|
||||
ALTER TABLE user_tokens ALTER COLUMN expiryDate DROP NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS event_subscription_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
drop table if exists alert_action_def;
|
||||
drop table if exists alert_entity;
|
||||
DELETE from entity_relationship where fromEntity = 'alert' and toEntity = 'alertAction';
|
||||
|
||||
-- create data model table
|
||||
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fullyQualifiedName)
|
||||
);
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,database}', '{connection,config,databaseName}', json#> '{connection,config,database}', true)
|
||||
WHERE servicetype = 'Druid' and json #>'{connection,config,database}' is not null;
|
||||
|
||||
-- We were using the same jsonSchema for Pipeline Services and Ingestion Pipeline status
|
||||
-- Also, we relied on the extension to store the run id
|
||||
UPDATE entity_extension_time_series
|
||||
SET jsonSchema = 'ingestionPipelineStatus', extension = 'ingestionPipeline.pipelineStatus'
|
||||
WHERE jsonSchema = 'pipelineStatus' AND extension <> 'pipeline.PipelineStatus';
|
||||
|
||||
-- We are refactoring the storage service with containers. We'll remove the locations
|
||||
DROP TABLE location_entity;
|
||||
DELETE FROM entity_relationship WHERE fromEntity='location' OR toEntity='location';
|
||||
TRUNCATE TABLE storage_service_entity;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,storageServiceName}'
|
||||
WHERE servicetype = 'Glue';
|
||||
|
||||
UPDATE chart_entity
|
||||
SET json = json::jsonb #- '{tables}';
|
||||
|
||||
-- Updating the tableau authentication fields
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSONB_SET(json::jsonb,
|
||||
'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}'||
|
||||
jsonb_build_object('authType',jsonb_build_object(
|
||||
'username',json #>'{connection,config,username}',
|
||||
'password',json #>'{connection,config,password}'
|
||||
)), true)
|
||||
where servicetype = 'Tableau'
|
||||
and json#>'{connection,config,password}' is not null
|
||||
and json#>'{connection,config,username}' is not null;
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSONB_SET(json::jsonb,
|
||||
'{connection,config}',json::jsonb #>'{connection,config}' #- '{personalAccessTokenName}' #- '{personalAccessTokenSecret}'||
|
||||
jsonb_build_object('authType',jsonb_build_object(
|
||||
'personalAccessTokenName',json #>'{connection,config,personalAccessTokenName}',
|
||||
'personalAccessTokenSecret',json #>'{connection,config,personalAccessTokenSecret}'
|
||||
)), true)
|
||||
where servicetype = 'Tableau'
|
||||
and json#>'{connection,config,personalAccessTokenName}' is not null
|
||||
and json#>'{connection,config,personalAccessTokenSecret}' is not null;
|
||||
|
||||
-- Removed property from metadataService.json
|
||||
UPDATE metadata_service_entity
|
||||
SET json = json::jsonb #- '{allowServiceCreation}'
|
||||
WHERE serviceType in ('Amundsen', 'Atlas', 'MetadataES', 'OpenMetadata');
|
||||
|
||||
UPDATE metadata_service_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"')
|
||||
WHERE name = 'OpenMetadata';
|
||||
|
||||
-- Fix Glue sample data endpoint URL to be a correct URI
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{connection,config,awsConfig,endPointURL}', '"https://glue.region_name.amazonaws.com/"')
|
||||
WHERE serviceType = 'Glue'
|
||||
AND json#>'{connection,config,awsConfig,endPointURL}' = '"https://glue.<region_name>.amazonaws.com/"';
|
||||
|
||||
-- Delete connectionOptions from superset
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = json::jsonb #- '{connection,config,connectionOptions}'
|
||||
WHERE serviceType = 'Superset';
|
||||
|
||||
-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,partitionQueryDuration}' #- '{connection,config,partitionQuery}' #- '{connection,config,partitionField}'
|
||||
WHERE serviceType = 'BigQuery';
|
||||
|
||||
-- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,supportsQueryComment}' #- '{connection,config,scheme}' #- '{connection,config,hostPort}' #- '{connection,config,supportsProfiler}'
|
||||
WHERE serviceType = 'Salesforce';
|
||||
|
||||
-- Delete supportsProfiler from DynamoDB
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,supportsProfiler}'
|
||||
WHERE serviceType = 'DynamoDB';
|
||||
|
||||
-- Update TagLabels source from 'Tag' to 'Classification' after #10486
|
||||
UPDATE table_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
|
||||
UPDATE ml_model_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
|
||||
|
||||
-- Delete uriString from Mssql
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,uriString}'
|
||||
WHERE serviceType = 'Mssql';
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
-- Updating the value of SASL Mechanism for Kafka and Redpanda connections
|
||||
UPDATE messaging_service_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{connection,config,saslMechanism}', '"PLAIN"')
|
||||
WHERE (servicetype = 'Kafka' OR serviceType = 'Redpanda')
|
||||
AND json#>'{connection,config,saslMechanism}' IS NOT NULL
|
||||
AND json#>'{connection,config,saslMechanism}' NOT IN ('"GSSAPI"', '"PLAIN"', '"SCRAM-SHA-256"', '"SCRAM-SHA-512"', '"OAUTHBEARER"');
|
||||
|
||||
-- Remove the Subscriptions
|
||||
DELETE FROM event_subscription_entity;
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
|
||||
-- Remove the ibmi scheme from Db2 replace it with db2+ibm_db
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{connection,config,scheme}', '"db2+ibm_db"')
|
||||
WHERE serviceType = 'Db2';
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
-- Update the tableau data model enum
|
||||
UPDATE dashboard_data_model_entity
|
||||
SET json = JSONB_SET(json::jsonb, '{dataModelType}', '"TableauDataModel"')
|
||||
WHERE json#>'{dataModelType}' = '"TableauSheet"';
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- use FQN instead of name for Test Connection Definition
|
||||
ALTER TABLE test_connection_definition
|
||||
ADD fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
DROP COLUMN name;
|
||||
|
||||
-- Since we are not deleting the test connection defs anymore, clean it up
|
||||
TRUNCATE test_connection_definition;
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
-- we are not using the secretsManagerCredentials
|
||||
UPDATE metadata_service_entity
|
||||
SET json = json::jsonb #- '{openMetadataServerConnection.secretsManagerCredentials}'
|
||||
where name = 'OpenMetadata';
|
||||
|
||||
-- Rename githubCredentials to gitCredentials
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,githubCredentials}', '{connection,config,gitCredentials}', json#>'{connection,config,githubCredentials}')
|
||||
where serviceType = 'Looker'
|
||||
and json#>'{connection,config,githubCredentials}' is not null;
|
||||
|
||||
-- Rename gcsConfig in BigQuery to gcpConfig
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,credentials,gcsConfig}', '{connection,config,credentials,gcpConfig}',
|
||||
json#>'{connection,config,credentials,gcsConfig}')
|
||||
where serviceType in ('BigQuery')
|
||||
and json#>'{connection,config,credentials,gcsConfig}' is not null;
|
||||
|
||||
-- Rename gcsConfig in Datalake to gcpConfig
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,configSource,securityConfig,gcsConfig}', '{connection,config,configSource,securityConfig,gcpConfig}',
|
||||
json#>'{connection,config,configSource,securityConfig,gcsConfig}')
|
||||
where serviceType in ('Datalake')
|
||||
and json#>'{connection,config,configSource,securityConfig,gcsConfig}' is not null;
|
||||
|
||||
-- Rename gcsConfig in dbt to gcpConfig
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}', '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}', (json#>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}')::jsonb)
|
||||
WHERE json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig}' is not null and json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}' is not null;
|
||||
|
||||
|
||||
-- Rename dashboardUrl in dashboard_entity to sourceUrl
|
||||
UPDATE dashboard_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{dashboardUrl}' , '{sourceUrl}',
|
||||
json#>'{dashboardUrl}')
|
||||
where json#>'{dashboardUrl}' is not null;
|
||||
|
||||
-- Rename chartUrl in chart_entity to sourceUr
|
||||
UPDATE chart_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{chartUrl}' , '{sourceUrl}',
|
||||
json#>'{chartUrl}')
|
||||
where json#>'{chartUrl}' is not null;
|
||||
|
||||
-- Rename pipelineUrl in pipeline_entity to sourceUrl
|
||||
UPDATE pipeline_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{pipelineUrl}' , '{sourceUrl}',
|
||||
json#>'{pipelineUrl}')
|
||||
where json#>'{pipelineUrl}' is not null;
|
||||
|
||||
|
||||
-- Rename taskUrl in pipeline_entity to sourceUrl
|
||||
UPDATE pipeline_entity
|
||||
SET json = jsonb_set(
|
||||
json::jsonb - 'tasks',
|
||||
'{tasks}',
|
||||
(
|
||||
SELECT jsonb_agg(
|
||||
jsonb_build_object(
|
||||
'name', t ->> 'name',
|
||||
'sourceUrl', t ->> 'taskUrl',
|
||||
'taskType', t ->> 'taskType',
|
||||
'description', t ->> 'description',
|
||||
'displayName', t ->> 'displayName',
|
||||
'fullyQualifiedName', t ->> 'fullyQualifiedName',
|
||||
'downstreamTasks', (t -> 'downstreamTasks')::jsonb,
|
||||
'tags', (t ->> 'tags')::jsonb,
|
||||
'endDate', t ->> 'endDate',
|
||||
'startDate', t ->> 'startDate',
|
||||
'taskSQL', t ->> 'taskSQL'
|
||||
)
|
||||
)
|
||||
FROM jsonb_array_elements(json->'tasks') AS t
|
||||
)
|
||||
)
|
||||
where json#>'{tasks}' is not null;
|
||||
|
||||
|
||||
-- Modify migrations for service connection of postgres and mysql to move password under authType
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,password}',
|
||||
'{connection,config,authType}',
|
||||
jsonb_build_object('password',json#>'{connection,config,password}')
|
||||
)
|
||||
WHERE serviceType IN ('Postgres', 'Mysql')
|
||||
and json#>'{connection,config,password}' is not null;
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
|
||||
-- Remove sourceUrl in pipeline_entity from DatabricksPipeline & Fivetran
|
||||
UPDATE pipeline_entity
|
||||
SET json = json::jsonb #- '{sourceUrl}'
|
||||
where json #> '{serviceType}' in ('"DatabricksPipeline"','"Fivetran"');
|
||||
|
||||
|
||||
-- Remove sourceUrl in dashboard_entity from Mode
|
||||
UPDATE dashboard_entity
|
||||
SET json = json::jsonb #- '{sourceUrl}'
|
||||
where json #> '{serviceType}' in ('"Mode"');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
|
||||
installed_rank SERIAL,
|
||||
version VARCHAR(256) PRIMARY KEY,
|
||||
migrationFileName VARCHAR(256) NOT NULL,
|
||||
checksum VARCHAR(256) NOT NULL,
|
||||
installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
|
||||
version VARCHAR(256) NOT NULL,
|
||||
sqlStatement VARCHAR(10000) NOT NULL,
|
||||
checksum VARCHAR(256) PRIMARY KEY,
|
||||
executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Update test definition parameterValues
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'columnValueMeanToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'columnValueMedianToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'columnValueStdDevToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minLength", "dataType": "INT", "required": false, "description": "The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length", "displayName": "Min", "optionValues": []}, {"name": "maxLength", "dataType": "INT", "required": false, "description": "The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'columnValueLengthsToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minValue", "dataType": "INT", "required": false, "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'columnValuesToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "columnNames", "dataType": "STRING", "required": true, "description": "Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string", "displayName": "Column Names", "optionValues": []}, {"name": "ordered", "dataType": "BOOLEAN", "required": false, "description": "Whether or not to considered the order of the list when performing the match check", "displayName": "Ordered", "optionValues": []}]'
|
||||
)
|
||||
where name = 'tableColumnToMatchSet';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name": "minValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
|
||||
)
|
||||
where name = 'tableRowCountToBeBetween';
|
||||
|
||||
update test_definition
|
||||
set json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
'[{"name":"sqlExpression","displayName":"SQL Expression","description":"SQL expression to run against the table","dataType":"STRING","required":"true"},{"name":"strategy","displayName":"Strategy","description":"Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to ROWS)","dataType":"ARRAY","optionValues":["ROWS","COUNT"],"required":false},{"name":"threshold","displayName":"Threshold","description":"Threshold to use to determine if the test passes or fails (defaults to 0).","dataType":"NUMBER","required":false}]'
|
||||
)
|
||||
where name = 'tableCustomSQLQuery';
|
||||
|
||||
-- Modify migrations for service connection of airflow to move password under authType if
|
||||
-- Connection Type as Mysql or Postgres
|
||||
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,connection,password}',
|
||||
'{connection,config,connection,authType}',
|
||||
jsonb_build_object('password',json#>'{connection,config,connection,password}')
|
||||
)
|
||||
WHERE serviceType = 'Airflow'
|
||||
and json#>'{connection,config,connection,type}' IN ('"Mysql"', '"Postgres"')
|
||||
and json#>'{connection,config,connection,password}' is not null;
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
-- Rename includeTempTables in snowflake to includeTransientTables
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(json::jsonb #- '{connection,config,includeTempTables}', '{connection,config,includeTransientTables}',
|
||||
json#>'{connection,config,includeTempTables}')
|
||||
where serviceType in ('Snowflake') and json#>'{connection,config,includeTempTables}' is not null ;
|
||||
|
||||
|
||||
update dbservice_entity
|
||||
set json = jsonb_set(json::jsonb, '{connection,config,scheme}', '"hive"')
|
||||
where json#>>'{connection,config,scheme}' in ('impala', 'impala4')
|
||||
and serviceType = 'Hive';
|
||||
|
||||
-- remove the dataModel references from Data Models
|
||||
UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}';
|
||||
|
||||
-- migrate ingestAllDatabases in mssql
|
||||
UPDATE dbservice_entity de2
|
||||
SET json = JSONB_SET(
|
||||
json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'||
|
||||
jsonb_build_object('database',
|
||||
(SELECT json->>'name'
|
||||
FROM database_entity de
|
||||
WHERE id = (SELECT er.toId
|
||||
FROM entity_relationship er
|
||||
WHERE er.fromId = de2.id
|
||||
AND er.toEntity = 'database'
|
||||
LIMIT 1)
|
||||
)
|
||||
)),
|
||||
'{connection,config,ingestAllDatabases}',
|
||||
'true'::jsonb
|
||||
)
|
||||
WHERE de2.serviceType = 'Mssql'
|
||||
AND json->>'{connection,config,database}' IS NULL;
|
||||
|
||||
-- remove keyfile from clickhouse
|
||||
UPDATE dbservice_entity
|
||||
SET json = json #-'{connection,config,keyfile}'
|
||||
WHERE serviceType = 'Clickhouse';
|
||||
|
||||
-- Clean old test connections
|
||||
TRUNCATE automations_workflow;
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
-- This column is already created in v000__create_server_change_log.sql
|
||||
-- ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics jsonb;
|
||||
Reference in New Issue
Block a user