chore: import upstream snapshot with attribution

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v000__create_server_change_log.sql

@@ -0,0 +1,20 @@
+-- Create tables for tracking server migrations
+-- This migration runs before all other migrations to ensure migration tracking tables exist
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    version VARCHAR(256) NOT NULL,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP,
+    metrics JSON,
+    PRIMARY KEY (version),
+    UNIQUE KEY installed_rank (installed_rank)
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v001__create_db_connection_info.sql

@@ -0,0 +1,421 @@
+--
+-- Table that captures all the relationships between entities
+--
+CREATE TABLE IF NOT EXISTS entity_relationship (
+    fromId VARCHAR(36) NOT NULL,                -- ID of the from entity
+    toId VARCHAR(36) NOT NULL,                  -- ID of the to entity
+    fromEntity VARCHAR(256) NOT NULL,           -- Type name of the from entity
+    toEntity VARCHAR(256) NOT NULL,             -- Type name of to entity
+    relation TINYINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSON,                                  -- JSON payload with additional information
+    deleted BOOLEAN NOT NULL DEFAULT FALSE,
+    INDEX edge_index (fromId, toId, relation),
+    INDEX from_index (fromId, relation),
+    INDEX to_index (toId, relation),
+    PRIMARY KEY (fromId, toId, relation)
+);
+
+--
+-- Table that captures all the relationships between field of an entity to a field of another entity
+-- Example - table1.column1 (fromFQN of type table.columns.column) is joined with(relation)
+-- table2.column8 (toFQN of type table.columns.column)
+--
+CREATE TABLE IF NOT EXISTS field_relationship (
+    fromFQN VARCHAR(256) NOT NULL,              -- Fully qualified name of entity or field
+    toFQN VARCHAR(256) NOT NULL,                -- Fully qualified name of entity or field
+    fromType VARCHAR(256) NOT NULL,             -- Fully qualified type of entity or field
+    toType VARCHAR(256) NOT NULL,               -- Fully qualified type of entity or field
+    relation TINYINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSON,                                  -- JSON payload with additional information
+    INDEX from_index (fromFQN, relation),
+    INDEX to_index (toFQN, relation),
+    PRIMARY KEY (fromFQN, toFQN, relation)
+);
+
+--
+-- Used for storing additional metadata for an entity
+--
+CREATE TABLE IF NOT EXISTS entity_extension (
+    id VARCHAR(36) NOT NULL,                    -- ID of the from entity
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSON NOT NULL,
+    PRIMARY KEY (id, extension)
+);
+
+--
+-- Service entities
+--
+CREATE TABLE IF NOT EXISTS dbservice_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS messaging_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS storage_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Data entities
+--
+CREATE TABLE IF NOT EXISTS database_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS database_schema_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS table_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS metric_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS report_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS ml_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS topic_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS chart_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS location_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL ,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Feed related tables
+--
+CREATE TABLE IF NOT EXISTS thread_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityId') NOT NULL,
+    entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL,
+    assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.addressedTo'),
+    json JSON NOT NULL,
+    createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.threadTs') STORED NOT NULL,
+    createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.createdBy') STORED NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    resolved BOOLEAN GENERATED ALWAYS AS (json -> '$.resolved'),
+    PRIMARY KEY (id)
+);
+
+--
+-- Policies related tables
+--
+CREATE TABLE IF NOT EXISTS policy_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Ingestion related tables
+--
+CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    timestamp BIGINT,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- User, Team, and bots
+--
+CREATE TABLE IF NOT EXISTS team_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS user_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    email VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.email') NOT NULL,
+    deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> '$.deactivated'),
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS bot_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS role_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    defaultRole BOOLEAN GENERATED ALWAYS AS (json -> '$.defaultRole'),
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Usage table where usage for all the entities is captured
+--
+CREATE TABLE IF NOT EXISTS entity_usage (
+    id VARCHAR(36) NOT NULL,         -- Unique id of the entity
+    entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
+    usageDate DATE,                  -- date corresponding to the usage
+    count1 INT,                      -- total daily count of use on usageDate
+    count7 INT,                      -- rolling count of last 7 days going back from usageDate
+    count30 INT,                     -- rolling count of last 30 days going back from usageDate
+    percentile1 INT,                 -- percentile rank with in same entity for given usage date
+    percentile7 INT,                 -- percentile rank with in same entity for last 7 days of usage
+    percentile30 INT,                -- percentile rank with in same entity for last 30 days of usage
+    UNIQUE (usageDate, id)
+);
+
+--
+-- Tag related tables
+--
+CREATE TABLE IF NOT EXISTS tag_category (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL, -- JSON stores category information and does not store children
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE (name) -- Unique tag category name
+);
+
+CREATE TABLE IF NOT EXISTS tag (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL, -- JSON stores all tag attributes and does not store children
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS tag_usage (
+    source TINYINT NOT NULL,            -- Source of the tag label
+    tagFQN VARCHAR(256) NOT NULL,       -- Fully qualified name of the tag
+    targetFQN VARCHAR(256) NOT NULL,    -- Fully qualified name of the entity instance or corresponding field
+    labelType TINYINT NOT NULL,         -- Type of tagging: manual, automated, propagated, derived
+    state TINYINT NOT NULL,             -- State of tagging: suggested or confirmed
+    UNIQUE (source, tagFQN, targetFQN)
+);
+
+CREATE TABLE IF NOT EXISTS change_event (
+    eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL,
+    userName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.userName') NOT NULL,
+    eventTime BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+    json JSON NOT NULL,
+    INDEX event_type_index (eventType),
+    INDEX entity_type_index (entityType),
+    INDEX event_time_index (eventTime)
+);
+
+CREATE TABLE IF NOT EXISTS webhook_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+    -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
+);
+
+CREATE TABLE IF NOT EXISTS glossary_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS glossary_term_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v002__create_db_connection_info.sql

@@ -0,0 +1,76 @@
+CREATE TABLE IF NOT EXISTS type_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    category VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.category') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+ALTER TABLE webhook_entity
+ADD status VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.status') NOT NULL,
+DROP COLUMN deleted;
+
+ALTER TABLE entity_relationship
+DROP INDEX edge_index;
+
+CREATE TABLE IF NOT EXISTS mlmodel_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE thread_entity SET json = JSON_SET(json, '$.type', 'Conversation', '$.reactions', JSON_ARRAY());
+
+ALTER TABLE thread_entity
+ADD type VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.type'),
+ADD taskId INT UNSIGNED GENERATED ALWAYS AS (json ->> '$.task.id'),
+ADD taskStatus VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.task.status'),
+ADD taskAssignees JSON GENERATED ALWAYS AS (json -> '$.task.assignees'),
+ADD CONSTRAINT task_id_constraint UNIQUE(taskId),
+ADD INDEX thread_type_index (type),
+ADD INDEX task_status_index (taskStatus),
+ADD INDEX created_by_index (createdBy),
+ADD INDEX updated_at_index (updatedAt);
+
+CREATE TABLE task_sequence (id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id));
+INSERT INTO task_sequence VALUES (0);
+
+
+DELETE from ingestion_pipeline_entity where 1=1;
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.database'),
+        '$.connection.config.databaseSchema',
+        JSON_EXTRACT(json, '$.connection.config.database')
+    ) where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle'); 
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.database'
+                    ,'$.connection.config.username'
+                    ,'$.connection.config.projectId'
+                    ,'$.connection.config.enablePolicyTagImport')
+WHERE serviceType = 'BigQuery';
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.database')
+WHERE serviceType in ('Athena','Databricks');
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler', '$.connection.config.pipelineServiceName')
+WHERE serviceType = 'Glue';
+
+UPDATE dashboard_service_entity
+SET json = JSON_REMOVE(json, '$.connection.config.dbServiceName')
+WHERE serviceType in ('Metabase','Superset','Tableau');
+
+DELETE FROM pipeline_service_entity WHERE 1=1;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v003__create_db_connection_info.sql

@@ -0,0 +1,59 @@
+DELETE from ingestion_pipeline_entity where 1=1;
+DELETE from entity_relationship where toEntity = 'ingestionPipeline';
+
+-- 0.10 had empty FQNs for services, users and teams
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE dashboard_service_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE messaging_service_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE pipeline_service_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE storage_service_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE team_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;
+
+UPDATE user_entity
+SET json = JSON_INSERT(
+         json,
+        '$.fullyQualifiedName',
+        JSON_EXTRACT(json, '$.name')
+    )
+WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') is NULL;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v004__create_db_connection_info.sql

@@ -0,0 +1,118 @@
+UPDATE team_entity
+SET json = JSON_INSERT(json, '$.teamType', 'Group');
+
+ALTER TABLE team_entity
+ADD teamType VARCHAR(64) GENERATED ALWAYS AS (json ->> '$.teamType') NOT NULL;
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.database')
+WHERE serviceType = 'DynamoDB';
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions')
+WHERE serviceType = 'DeltaLake';
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler')
+WHERE serviceType = 'DeltaLake';
+
+UPDATE dashboard_service_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.username'),
+        '$.connection.config.clientId',
+        JSON_EXTRACT(json, '$.connection.config.username')
+    )
+WHERE serviceType = 'Looker';
+
+UPDATE dashboard_service_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.password'),
+        '$.connection.config.clientSecret',
+        JSON_EXTRACT(json, '$.connection.config.password')
+    )
+WHERE serviceType = 'Looker';
+
+UPDATE dashboard_service_entity
+SET json = JSON_REMOVE(json, '$.connection.config.env')
+WHERE serviceType = 'Looker';
+
+CREATE TABLE IF NOT EXISTS test_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityType') NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_suite (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_case (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    entityFQN VARCHAR (1024) GENERATED ALWAYS AS (json ->> '$.entityFQN') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE webhook_entity
+SET json = JSON_INSERT(json, '$.webhookType', 'generic');
+
+ALTER TABLE webhook_entity
+ADD webhookType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.webhookType') NOT NULL;
+
+CREATE TABLE IF NOT EXISTS entity_extension_time_series (
+    entityFQN VARCHAR(1024) NOT NULL,           -- Entity FQN, we can refer to tables and columns
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSON NOT NULL,
+    timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL
+);
+
+ALTER TABLE thread_entity
+    ADD announcementStart BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.announcement.startTime'),
+    ADD announcementEnd BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.announcement.endTime');
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.databaseSchema','$.connection.config.oracleServiceName')
+WHERE serviceType = 'Oracle';
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.hostPort')
+WHERE serviceType = 'Athena';
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.username', '$.connection.config.password')
+WHERE serviceType in ('Databricks');
+
+CREATE TABLE IF NOT EXISTS openmetadata_settings (
+    id MEDIUMINT NOT NULL AUTO_INCREMENT,
+    configType VARCHAR(36) NOT NULL,
+    json JSON NOT NULL,
+    PRIMARY KEY (id, configType),
+    UNIQUE(configType)
+);
+
+DELETE FROM entity_extension 
+WHERE jsonSchema IN ('tableProfile', 'columnTest', 'tableTest');
+
+DELETE FROM ingestion_pipeline_entity 
+WHERE LOWER(JSON_EXTRACT(json, '$.pipelineType') = 'profiler');
+
+DELETE FROM role_entity;
+DELETE FROM policy_entity;
+DELETE FROM field_relationship WHERE fromType IN ('role', 'policy') OR toType IN ('role', 'policy');
+DELETE FROM entity_relationship WHERE fromEntity IN ('role', 'policy') OR toEntity IN ('role', 'policy');
+ALTER TABLE role_entity DROP COLUMN defaultRole;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v005__create_db_connection_info.sql

@@ -0,0 +1,305 @@
+DELETE FROM entity_relationship
+WHERE toEntity = 'ingestionPipeline'
+AND toId NOT IN (
+	SELECT DISTINCT id 
+	FROM ingestion_pipeline_entity
+);
+
+CREATE TABLE IF NOT EXISTS user_tokens (
+    token VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.token') STORED NOT NULL,
+    userId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.userId') STORED NOT NULL,
+    tokenType VARCHAR(50) GENERATED ALWAYS AS (json ->> '$.tokenType') STORED NOT NULL,
+    json JSON NOT NULL,
+    expiryDate BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.expiryDate') NOT NULL,
+    PRIMARY KEY (token)
+);
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.metastoreHostPort'),
+        '$.connection.config.metastoreConnection',
+        JSON_OBJECT('metastoreHostPort', JSON_EXTRACT(json, '$.connection.config.metastoreHostPort'))
+    )
+where serviceType = 'DeltaLake'
+  and JSON_EXTRACT(json, '$.connection.config.metastoreHostPort') is not null;
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.metastoreFilePath'),
+        '$.connection.config.metastoreConnection',
+        JSON_OBJECT('metastoreFilePath', JSON_EXTRACT(json, '$.connection.config.metastoreFilePath'))
+    )
+where serviceType = 'DeltaLake'
+  and JSON_EXTRACT(json, '$.connection.config.metastoreFilePath') is not null;
+
+
+ALTER TABLE test_definition 
+ADD COLUMN supported_data_types JSON GENERATED ALWAYS AS (json -> '$.supportedDataTypes');
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableColumnCountToEqual'
+)
+WHERE BINARY name = 'TableColumnCountToEqual';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableColumnCountToEqual'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableColumnCountToEqual"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableColumnToMatchSet'
+)
+WHERE BINARY name = 'TableColumnToMatchSet';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableColumnToMatchSet'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableColumnToMatchSet"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableColumnNameToExist'
+)
+WHERE BINARY name = 'TableColumnNameToExist';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableColumnNameToExist'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableColumnNameToExist"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableRowCountToBeBetween'
+)
+WHERE BINARY name = 'TableRowCountToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableRowCountToBeBetween'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableRowCountToBeBetween"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableColumnCountToBeBetween'
+)
+WHERE BINARY name = 'TableColumnCountToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableColumnCountToBeBetween'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableColumnCountToBeBetween"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'columnValuesToBeInSet'
+)
+WHERE BINARY name = 'ColumnValuesToBeInSet';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'columnValuesToBeInSet'
+)
+WHERE BINARY JSON_CONTAINS(json, '"ColumnValuesToBeInSet"', '$.fullyQualifiedName');
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.name'),
+	'$.name',
+	'tableRowCountToEqual'
+)
+WHERE BINARY name = 'TableRowCountToEqual';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.fullyQualifiedName'),
+	'$.fullyQualifiedName',
+	'tableRowCountToEqual'
+)
+WHERE BINARY JSON_CONTAINS(json, '"TableRowCountToEqual"', '$.fullyQualifiedName');
+
+
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValueMaxToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'ARRAY', 'SET')
+)
+WHERE name = 'columnValueMeanToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValueMedianToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValueMinToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR', 'ARRAY')
+)
+WHERE name = 'columnValueLengthsToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
+)
+WHERE name = 'columnValuesMissingCount'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValuesSumToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValueStdDevToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT')
+)
+WHERE name = 'columnValuesToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
+)
+WHERE name = 'columnValuesToBeInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
+)
+WHERE name = 'columnValuesToBeNotInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
+)
+WHERE name = 'columnValuesToBeNotNull'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER','TINYINT','SMALLINT','INT','BIGINT','BYTEINT','BYTES','FLOAT','DOUBLE','DECIMAL','NUMERIC','TIMESTAMP','TIMESTAMPZ','TIME','DATE','DATETIME','INTERVAL','STRING','MEDIUMTEXT','TEXT','CHAR','VARCHAR','BOOLEAN','BINARY','VARBINARY','ARRAY','BLOB','LONGBLOB','MEDIUMBLOB','MAP','STRUCT','UNION','SET','GEOGRAPHY','ENUM','JSON','UUID','VARIANT','GEOMETRY','POINT','POLYGON')
+)
+WHERE name = 'columnValuesToBeUnique'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
+)
+WHERE name = 'columnValuesToMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	json,
+	'$.supportedDataTypes',
+	JSON_ARRAY('BYTES', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR')
+)
+WHERE name = 'columnValuesToNotMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE pipeline_service_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.dbConnection')
+WHERE serviceType = 'Dagster';  
+
+
+UPDATE pipeline_service_entity 
+SET JSON = JSON_INSERT(
+	JSON_REMOVE(json, '$.connection.config.type','$.serviceType'),
+		'$.connection.config.type','GluePipeline',
+		'$.serviceType','GluePipeline'
+	)
+where serviceType='Glue';

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql

@@ -0,0 +1,126 @@
+--
+-- Upgrade changes for 0.13
+--
+ALTER TABLE `entity_extension_time_series` modify entityFQN varchar(768);
+ALTER TABLE `entity_extension_time_series` ADD INDEX `entity_fqn_index` (`entityFQN`);
+
+CREATE TABLE IF NOT EXISTS web_analytic_event (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.eventType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE(name),
+    INDEX name_index (name)
+);
+
+UPDATE bot_entity
+SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system');
+
+CREATE TABLE IF NOT EXISTS data_insight_chart (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.dataIndexType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    UNIQUE(name),
+    INDEX name_index (name)
+);
+
+UPDATE role_entity
+SET json = JSON_INSERT(json, '$.provider', 'system')
+WHERE name in ('DataConsumer', 'DataSteward');
+
+UPDATE policy_entity
+SET json = JSON_INSERT(json, '$.provider', 'system')
+WHERE fullyQualifiedName in (json, 'DataConsumerPolicy', 'DataStewardPolicy', 'OrganizationPolicy', 'TeamOnlyPolicy');
+
+UPDATE tag_category
+SET json = JSON_INSERT(json, '$.provider', 'system')
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = JSON_INSERT(json, '$.provider', 'system')
+WHERE fullyQualifiedName in ('PersonalData.Personal', 'PersonalData.SpecialCategory',
+'PII.None', 'PII.NonSensitive', 'PII.Sensitive',
+'Tier.Tier1', 'Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5');
+
+UPDATE pipeline_service_entity 
+SET json = JSON_INSERT(json ,'$.connection.config.configSource.hostPort', '$.connection.config.hostPort')
+WHERE serviceType = 'Dagster';
+
+UPDATE pipeline_service_entity 
+SET json = JSON_REMOVE(json ,'$.connection.config.hostPort', '$.connection.config.numberOfStatus')
+WHERE serviceType = 'Dagster';
+
+-- Remove categoryType
+UPDATE tag_category
+SET json = JSON_REMOVE(json ,'$.categoryType');
+
+-- Set mutuallyExclusive flag
+UPDATE tag_category
+SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'false');
+
+UPDATE tag_category
+SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'true')
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = JSON_INSERT(json ,'$.mutuallyExclusive', 'false');
+
+-- Merge mutually exclusive table when multiple tag labels are used
+DELETE t1 FROM tag_usage t1
+INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
+WHERE (t2.tagFQN = 'Tier.Tier1' AND t1.tagFQN in ('Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')) OR
+(t2.tagFQN = 'Tier.Tier2' AND t1.tagFQN in ('Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')) OR
+(t2.tagFQN = 'Tier.Tier3' AND t1.tagFQN in ('Tier.Tier4', 'Tier.Tier5')) OR
+(t2.tagFQN = 'Tier.Tier4' AND t1.tagFQN in ('Tier.Tier5'));
+
+DELETE t1 FROM tag_usage t1
+INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
+WHERE (t2.tagFQN = 'PII.Sensitive' AND t1.tagFQN in ('PII.NonSensitive', 'PII.None')) OR
+(t2.tagFQN = 'PII.NonSensitive' AND t1.tagFQN in ('PII.None'));
+
+DELETE t1 FROM tag_usage t1
+INNER JOIN tag_usage t2 ON (t1.targetFQN = t2.targetFQN)
+WHERE (t2.tagFQN = 'PersonalData.Personal' AND t1.tagFQN in ('PersonalData.SpecialCategory'));
+
+CREATE TABLE IF NOT EXISTS kpi_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS metadata_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- We are starting to store the current deployed flag. Let's mark it as false by default
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.deployed');
+
+UPDATE ingestion_pipeline_entity
+SET json = JSON_INSERT(json ,'$.deployed', 'true');
+
+-- We removed the supportsMetadataExtraction field in the `OpenMetadataConnection` object being used in IngestionPipelines
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.openMetadataServerConnection.supportsMetadataExtraction');

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v007__create_db_connection_info.sql

@@ -0,0 +1,67 @@
+-- Remove markDeletedTablesFromFilterOnly 
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.sourceConfig.config.markDeletedTablesFromFilterOnly');
+
+UPDATE data_insight_chart 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.dimensions'),
+	'$.dimensions',
+	JSON_ARRAY(
+		JSON_OBJECT('name', 'entityFqn', 'chartDataType', 'STRING'),
+		JSON_OBJECT('name', 'owner', 'chartDataType', 'STRING'),
+		JSON_OBJECT('name', 'entityType', 'chartDataType', 'STRING'),
+		JSON_OBJECT('name', 'entityHref', 'chartDataType', 'STRING')
+		)
+)
+WHERE name = 'mostViewedEntities';
+
+DROP TABLE webhook_entity;
+
+CREATE TABLE IF NOT EXISTS alert_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+    -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
+);
+
+CREATE TABLE IF NOT EXISTS alert_action_def (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    alertActionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.alertActionType') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.databaseSchema'),
+        '$.connection.config.database',
+        JSON_EXTRACT(json, '$.connection.config.databaseSchema')
+    ) where serviceType in ('Db2');
+
+DELETE from openmetadata_settings where configType = 'activityFeedFilterSetting';
+
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.sourceConfig.config.dbtConfigSource');
+
+UPDATE pipeline_service_entity
+SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.host')),'$.connection.config.token',JSON_EXTRACT(json, '$.connection.config.configSource.token'))
+WHERE  serviceType = 'Dagster' AND json -> '$.connection.config.configSource.host' IS NOT NULL;
+
+UPDATE pipeline_service_entity
+SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.hostPort')), '$.connection.config.token','')
+WHERE  serviceType = 'Dagster' AND json -> '$.connection.config.configSource.hostPort' IS NOT NULL;
+
+UPDATE topic_entity
+SET json = JSON_INSERT(JSON_REMOVE(json, '$.schemaType'), '$.messageSchema', JSON_OBJECT('schemaType', JSON_EXTRACT(json, '$.schemaType')))
+WHERE json -> '$.schemaType' IS NOT NULL;
+
+UPDATE topic_entity
+SET json = JSON_INSERT(JSON_REMOVE(json, '$.schemaText'), '$.messageSchema.schemaText', JSON_EXTRACT(json, '$.schemaText'))
+WHERE json -> '$.schemaText' IS NOT NULL;
+

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v008__create_db_connection_info.sql

@@ -0,0 +1,55 @@
+RENAME TABLE tag_category TO classification;
+
+-- Rename tagCategoryName in BigQuery for classificationName
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.tagCategoryName'),
+    '$.connection.config.classificationName',
+    JSON_EXTRACT(json, '$.connection.config.tagCategoryName')
+) where serviceType in ('BigQuery');
+
+-- Delete supportsUsageExtraction from vertica
+UPDATE dbservice_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.supportsUsageExtraction')
+WHERE serviceType = 'Vertica';
+
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json ,'$.sourceConfig.config.dbtConfigSource.dbtUpdateDescriptions')
+WHERE json -> '$.sourceConfig.config.type' = 'DBT';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.supportedDataTypes'),
+	'$.supportedDataTypes',
+	JSON_ARRAY('NUMBER', 'INT', 'FLOAT', 'DOUBLE', 'DECIMAL', 'TINYINT', 'SMALLINT', 'BIGINT', 'BYTEINT', 'TIMESTAMP', 'TIMESTAMPZ','DATETIME', 'DATE')
+)
+WHERE name = 'columnValuesToBeBetween';
+
+UPDATE pipeline_entity 
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.name'),
+        '$.name',
+		REPLACE(JSON_UNQUOTE(JSON_EXTRACT(json, '$.name')),':','')
+    )
+WHERE JSON_EXTRACT(json, '$.serviceType') = 'Dagster';
+
+UPDATE pipeline_entity 
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.fullyQualifiedName'),
+        '$.fullyQualifiedName',
+		REPLACE(JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')),':','')
+    )
+WHERE JSON_EXTRACT(json, '$.serviceType') = 'Dagster';
+
+
+UPDATE dashboard_service_entity  
+SET json = JSON_INSERT(
+JSON_REMOVE(json,'$.connection.config.username','$.connection.config.password','$.connection.config.provider'),
+'$.connection.config.connection',
+JSON_OBJECT(
+	'username',JSON_EXTRACT(json,'$.connection.config.username'),
+	'password',JSON_EXTRACT(json,'$.connection.config.password'),
+	'provider',JSON_EXTRACT(json,'$.connection.config.provider')
+	)
+)
+WHERE serviceType = 'Superset';

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

@@ -0,0 +1,232 @@
+-- Unique constraint for user email address
+ALTER TABLE user_entity
+ADD UNIQUE (email);
+
+
+-- Remove classificationName in BigQuery
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.classificationName') where serviceType in ('BigQuery');
+
+-- migrate ingestAllDatabases in postgres 
+UPDATE dbservice_entity de2 
+SET json = JSON_REPLACE(
+    JSON_INSERT(json, 
+      '$.connection.config.database', 
+      (select JSON_EXTRACT(json, '$.name') 
+        from database_entity de 
+        where id = (select er.toId 
+            from entity_relationship er 
+            where er.fromId = de2.id 
+              and er.toEntity = 'database' 
+            LIMIT 1
+          ))
+    ), '$.connection.config.ingestAllDatabases', 
+    true
+  ) 
+where de2.serviceType = 'Postgres' 
+  and JSON_EXTRACT(json, '$.connection.config.database') is NULL;
+
+CREATE TABLE IF NOT EXISTS storage_container_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS test_connection_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS automations_workflow (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.workflowType') STORED NOT NULL,
+    status VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.status') STORED,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- Do not store OM server connection, we'll set it dynamically on the resource
+UPDATE ingestion_pipeline_entity
+SET json = JSON_REMOVE(json, '$.openMetadataServerConnection');
+
+CREATE TABLE IF NOT EXISTS query_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE(name),
+    INDEX name_index (name)
+);
+
+CREATE TABLE IF NOT EXISTS temp_query_migration (
+    tableId VARCHAR(36)NOT NULL,
+    queryId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
+    queryName VARCHAR(255) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    json JSON NOT NULL
+);
+
+
+INSERT INTO temp_query_migration(tableId,json)
+SELECT id,JSON_OBJECT('id',UUID(),'query',query,'users',users,'checksum',checksum,'duration',duration,'name',checksum,'updatedAt',UNIX_TIMESTAMP(NOW()),'updatedBy','admin','deleted',false) as json from entity_extension d, json_table(d.json, '$[*]' columns (query varchar(200) path '$.query',users json path '$.users',checksum varchar(200) path '$.checksum',name varchar(255) path '$.checksum', duration double path '$.duration',queryDate varchar(200) path '$.queryDate')) AS j WHERE extension = 'table.tableQueries';
+
+INSERT INTO query_entity (json)
+SELECT t.json from temp_query_migration t
+ON DUPLICATE KEY UPDATE json = VALUES(json);
+
+INSERT INTO entity_relationship(fromId,toId,fromEntity,toEntity,relation)
+SELECT tmq.tableId, (select qe.id from query_entity qe where qe.name = tmq.queryName) ,'table','query',5 FROM temp_query_migration tmq;
+
+DELETE FROM entity_extension WHERE id IN
+ (SELECT DISTINCT tableId FROM temp_query_migration) AND extension = 'table.tableQueries';
+
+DROP Table temp_query_migration;
+
+-- remove the audience if it was wrongfully sent from the UI after editing the OM service
+UPDATE metadata_service_entity
+SET json = JSON_REMOVE(json, '$.connection.config.securityConfig.audience')
+WHERE name = 'OpenMetadata' AND JSON_EXTRACT(json, '$.connection.config.authProvider') != 'google';
+
+ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.expiryDate');
+
+CREATE TABLE IF NOT EXISTS event_subscription_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+    -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
+);
+
+drop table if exists alert_action_def;
+drop table if exists alert_entity;
+DELETE from entity_relationship where  fromEntity = 'alert' and toEntity = 'alertAction';
+
+-- create data model table
+CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.database'),
+        '$.connection.config.databaseName', JSON_EXTRACT(json, '$.connection.config.database')
+    )
+where serviceType = 'Druid'
+  and JSON_EXTRACT(json, '$.connection.config.database') is not null;
+
+-- We were using the same jsonSchema for Pipeline Services and Ingestion Pipeline status
+-- Also, we relied on the extension to store the run id
+UPDATE entity_extension_time_series
+SET jsonSchema = 'ingestionPipelineStatus', extension = 'ingestionPipeline.pipelineStatus'
+WHERE jsonSchema = 'pipelineStatus' AND extension <> 'pipeline.PipelineStatus';
+
+-- We are refactoring the storage service with containers. We'll remove the locations
+DROP TABLE location_entity;
+DELETE FROM entity_relationship WHERE fromEntity='location' OR toEntity='location';
+TRUNCATE TABLE storage_service_entity;
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.storageServiceName')
+WHERE serviceType = 'Glue';
+
+UPDATE chart_entity
+SET json = JSON_REMOVE(json, '$.tables');
+
+-- Updating the tableau authentication fields
+UPDATE dashboard_service_entity  
+SET json = JSON_INSERT(
+JSON_REMOVE(json,'$.connection.config.username','$.connection.config.password'),
+'$.connection.config.authType',
+JSON_OBJECT(
+	'username',JSON_EXTRACT(json,'$.connection.config.username'),
+	'password',JSON_EXTRACT(json,'$.connection.config.password')
+	)
+)
+WHERE serviceType = 'Tableau'
+AND JSON_EXTRACT(json, '$.connection.config.username') is not null
+AND JSON_EXTRACT(json, '$.connection.config.password') is not null;
+
+UPDATE dashboard_service_entity  
+SET json = JSON_INSERT(
+JSON_REMOVE(json,'$.connection.config.personalAccessTokenName','$.connection.config.personalAccessTokenSecret'),
+'$.connection.config.authType',
+JSON_OBJECT(
+	'personalAccessTokenName',JSON_EXTRACT(json,'$.connection.config.personalAccessTokenName'),
+	'personalAccessTokenSecret',JSON_EXTRACT(json,'$.connection.config.personalAccessTokenSecret')
+	)
+)
+WHERE serviceType = 'Tableau'
+AND JSON_EXTRACT(json, '$.connection.config.personalAccessTokenName') is not null
+AND JSON_EXTRACT(json, '$.connection.config.personalAccessTokenSecret') is not null;
+
+-- Removed property from metadataService.json
+UPDATE metadata_service_entity
+SET json = JSON_REMOVE(json, '$.allowServiceCreation')
+WHERE serviceType in ('Amundsen', 'Atlas', 'MetadataES', 'OpenMetadata');
+
+UPDATE metadata_service_entity
+SET json = JSON_INSERT(json, '$.provider', 'system')
+WHERE name = 'OpenMetadata';
+
+-- Fix Glue sample data endpoint URL to be a correct URI
+UPDATE dbservice_entity
+SET json = JSON_REPLACE(json, '$.connection.config.awsConfig.endPointURL', 'https://glue.region_name.amazonaws.com/')
+WHERE serviceType = 'Glue'
+  AND JSON_EXTRACT(json, '$.connection.config.awsConfig.endPointURL') = 'https://glue.<region_name>.amazonaws.com/';
+
+-- Delete connectionOptions from superset
+UPDATE dashboard_service_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions')
+WHERE serviceType = 'Superset';
+
+-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
+UPDATE dbservice_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.partitionQueryDuration', '$.connection.config.partitionQuery', '$.connection.config.partitionField')
+WHERE serviceType = 'BigQuery'; 
+
+-- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
+UPDATE dbservice_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.scheme', '$.connection.config.hostPort', '$.connection.config.supportsProfiler', '$.connection.config.supportsQueryComment')
+WHERE serviceType = 'Salesforce';
+
+-- Delete supportsProfiler from DynamoDB
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.supportsProfiler')
+WHERE serviceType = 'DynamoDB';
+
+-- Update TagLabels source from 'Tag' to 'Classification' after #10486
+UPDATE table_entity SET json = REGEXP_REPLACE(json, '\"source\"\\s*:\\s*\"Tag\"', '\"source\": \"Classification\"');
+UPDATE ml_model_entity SET json = REGEXP_REPLACE(json, '\"source\"\\s*:\\s*\"Tag\"', '\"source\": \"Classification\"');
+
+
+-- Delete supportsProfiler from Mssql
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.uriString')
+WHERE serviceType = 'Mssql';

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v010__create_db_connection_info.sql

@@ -0,0 +1,17 @@
+-- Updating the value of SASL Mechanism for Kafka and Redpanda connections
+UPDATE messaging_service_entity
+SET json = JSON_REPLACE(json, '$.connection.config.saslMechanism', 'PLAIN')
+WHERE (serviceType = 'Kafka' OR serviceType = 'Redpanda')
+  AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') IS NOT NULL
+  AND JSON_EXTRACT(json, '$.connection.config.saslMechanism') NOT IN ('GSSAPI', 'PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512', 'OAUTHBEARER');
+
+-- Remove the Subscriptions
+DELETE FROM event_subscription_entity;
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove the ibmi scheme from Db2 replace it with db2+ibm_db
+UPDATE dbservice_entity
+SET json = JSON_SET(json, '$.connection.config.scheme', 'db2+ibm_db')
+WHERE serviceType  = 'Db2';

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v011__create_db_connection_info.sql

@@ -0,0 +1,4 @@
+-- Update the tableau data model enum
+UPDATE dashboard_data_model_entity 
+SET json = JSON_SET(json, '$.dataModelType', 'TableauDataModel')
+WHERE JSON_EXTRACT(json, '$.dataModelType') = 'TableauSheet';

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v012__create_db_connection_info.sql

@@ -0,0 +1,7 @@
+-- use FQN instead of name for Test Connection Definition
+ALTER TABLE test_connection_definition
+ADD fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
+DROP COLUMN name;
+
+-- Since we are not deleting the test connection defs anymore, clean it up
+TRUNCATE TABLE test_connection_definition;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql

@@ -0,0 +1,363 @@
+-- we are not using the secretsManagerCredentials
+UPDATE metadata_service_entity
+SET json = JSON_REMOVE(json, '$.openMetadataServerConnection.secretsManagerCredentials')
+where name = 'OpenMetadata';
+
+-- Rename githubCredentials to gitCredentials
+UPDATE dashboard_service_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.githubCredentials'),
+        '$.connection.config.gitCredentials',
+        JSON_EXTRACT(json, '$.connection.config.githubCredentials')
+    )
+WHERE serviceType = 'Looker'
+  AND JSON_EXTRACT(json, '$.connection.config.githubCredentials') IS NOT NULL;
+
+
+-- Rename gcsConfig in BigQuery to gcpConfig
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.credentials.gcsConfig'),
+    '$.connection.config.credentials.gcpConfig',
+    JSON_EXTRACT(json, '$.connection.config.credentials.gcsConfig')
+) where serviceType in ('BigQuery');
+
+-- Rename gcsConfig in Datalake to gcpConfig
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.configSource.securityConfig.gcsConfig'),
+    '$.connection.config.configSource.securityConfig.gcpConfig',
+    JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcsConfig')
+) where serviceType in ('Datalake')
+AND JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcsConfig') IS NOT NULL;
+
+
+-- Rename gcsConfig in dbt to gcpConfig
+UPDATE ingestion_pipeline_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig'),
+    '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig',
+    JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig')
+)
+WHERE json -> '$.sourceConfig.config.type' = 'DBT'
+AND JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcsConfig') IS NOT NULL;
+
+-- Rename chartUrl in chart_entity to sourceUrl
+UPDATE chart_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.chartUrl'),
+        '$.sourceUrl',
+        JSON_EXTRACT(json, '$.chartUrl')
+    )
+WHERE JSON_EXTRACT(json, '$.chartUrl') IS NOT NULL;
+
+-- Rename dashboardUrl in dashboard_entity to sourceUrl
+UPDATE dashboard_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.dashboardUrl'),
+        '$.sourceUrl',
+        JSON_EXTRACT(json, '$.dashboardUrl')
+    )
+WHERE JSON_EXTRACT(json, '$.dashboardUrl') IS NOT NULL;
+
+-- Rename pipelineUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.pipelineUrl'),
+        '$.sourceUrl',
+        JSON_EXTRACT(json, '$.pipelineUrl')
+    )
+WHERE JSON_EXTRACT(json, '$.pipelineUrl') IS NOT NULL;
+
+
+-- Rename taskUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity AS pe
+JOIN (
+    SELECT id, JSON_ARRAYAGG(JSON_OBJECT(
+        'name', t.name,
+        'sourceUrl', t.sourceUrl,
+        'taskType', t.taskType,
+        'description', t.description,
+        'displayName', t.displayName,
+        'fullyQualifiedName', t.fullyQualifiedName,
+        'downstreamTasks', t.downstreamTasks,
+        'tags', t.tags,
+        'endDate', t.endDate,
+        'startDate', t.startDate,
+        'taskSQL', t.taskSQL
+    )) AS updated_json
+    FROM pipeline_entity,
+    JSON_TABLE(
+        json,
+        '$.tasks[*]' COLUMNS (
+            name VARCHAR(256) PATH '$.name',
+            sourceUrl VARCHAR(256) PATH '$.taskUrl',
+            taskType VARCHAR(256) PATH '$.taskType',
+            description TEXT PATH '$.description',
+            displayName VARCHAR(256) PATH '$.displayName',
+            fullyQualifiedName VARCHAR(256) PATH '$.fullyQualifiedName',
+            downstreamTasks JSON PATH '$.downstreamTasks',
+            tags JSON PATH '$.tags',
+            endDate VARCHAR(256) PATH '$.endDate',
+            startDate VARCHAR(256) PATH '$.startDate',
+            taskSQL TEXT PATH '$.taskSQL'
+        )
+    ) AS t
+    GROUP BY id
+) AS updated_data
+ON pe.id = updated_data.id
+SET pe.json = JSON_INSERT(
+    JSON_REMOVE(pe.json, '$.tasks'),
+    '$.tasks',
+    updated_data.updated_json
+);
+
+-- Modify migrations for service connection of postgres and mysql to move password under authType
+
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.password'),
+    '$.connection.config.authType',
+    JSON_OBJECT(),
+    '$.connection.config.authType.password',
+    JSON_EXTRACT(json, '$.connection.config.password'))
+where serviceType in ('Postgres', 'Mysql');
+
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove sourceUrl in pipeline_entity from DatabricksPipeline & Fivetran
+UPDATE pipeline_entity
+SET json = JSON_REMOVE(json, '$.sourceUrl')
+WHERE JSON_EXTRACT(json, '$.serviceType') in ('DatabricksPipeline','Fivetran');
+
+-- Remove sourceUrl in dashboard_entity from Mode
+UPDATE dashboard_entity 
+SET json = JSON_REMOVE(json, '$.sourceUrl')
+WHERE JSON_EXTRACT(json, '$.serviceType') in ('Mode');
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank SERIAL,
+    version VARCHAR(256)  PRIMARY KEY,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Update test definition parameterValues
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+			'name', 'minValueForMeanInCol',
+			'dataType', 'INT',
+			'required', false,
+			'description', 'Expected mean value for the column to be greater or equal than',
+			'displayName', 'Min'
+		),
+		JSON_OBJECT(
+			'name', 'maxValueForMeanInCol',
+			'dataType', 'INT',
+			'required', false,
+			'description', 'Expected mean value for the column to be lower or equal than',
+			'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'columnValueMeanToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+	        'name', 'minValueForMedianInCol',
+	        'dataType', 'INT',
+	        'required', false,
+	        'description', 'Expected median value for the column to be greater or equal than',
+	        'displayName', 'Min'
+		),
+		JSON_OBJECT(
+        'name', 'maxValueForMedianInCol',
+        'dataType', 'INT',
+        'required', false,
+        'description', 'Expected median value for the column to be lower or equal than',
+        'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'columnValueMedianToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+	        'name', 'minValueForStdDevInCol',
+	        'dataType', 'INT',
+	        'required', false,
+	        'description', 'Expected std. dev value for the column to be greater or equal than',
+	        'displayName', 'Min'
+		),
+		JSON_OBJECT(
+	        'name', 'maxValueForStdDevInCol',
+	        'dataType', 'INT',
+	        'required', false,
+	        'description', 'Expected std. dev value for the column to be lower or equal than',
+	        'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'columnValueStdDevToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+	        'name', 'minLength',
+	        'dataType', 'INT',
+	        'required', false,
+	        'description', 'The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length',
+	        'displayName', 'Min'
+		),
+		JSON_OBJECT(
+	        'name', 'maxLength',
+	        'dataType', 'INT',
+	        'required', false,
+	        'description', 'The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length',
+	        'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'columnValueLengthsToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+        'name', 'minValue',
+        'dataType', 'INT',
+        'required', false,
+        'description', 'The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum',
+        'displayName', 'Min'
+		),
+		JSON_OBJECT(
+        'name', 'maxValue',
+        'dataType', 'INT',
+        'required', false,
+        'description', 'The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum',
+        'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'columnValuesToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+        'name', 'columnNames',
+        'dataType', 'STRING',
+        'required', true,
+        'description', 'Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string',
+        'displayName', 'Column Names'
+		),
+		JSON_OBJECT(
+        'name', 'ordered',
+        'dataType', 'BOOLEAN',
+        'required', false,
+        'description', 'Whether or not to considered the order of the list when performing the match check',
+        'displayName', 'Ordered'
+		)
+	)	
+)
+WHERE name = 'tableColumnToMatchSet';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+        'name', 'minValue',
+        'dataType', 'INT',
+        'required', false,
+        'description', 'Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum',
+        'displayName', 'Min'
+		),
+		JSON_OBJECT(
+        'name', 'maxValue',
+        'dataType', 'INT',
+        'required', false,
+        'description', 'Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum',
+        'displayName', 'Max'
+		)
+	)	
+)
+WHERE name = 'tableRowCountToBeBetween';
+
+UPDATE test_definition 
+SET json = JSON_INSERT(
+	JSON_REMOVE(json, '$.parameterDefinition'),
+	'$.parameterDefinition',
+	JSON_ARRAY(
+		JSON_OBJECT(
+		     'name', 'sqlExpression',
+        	'displayName', 'SQL Expression',
+        	'description', 'SQL expression to run against the table',
+        	'dataType', 'STRING',
+        	'required', 'true'
+		),
+		JSON_OBJECT(
+			'name', 'strategy',
+	        'displayName', 'Strategy',
+    	    'description', 'Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to ROWS)',
+        	'dataType', 'ARRAY',
+	        'optionValues', JSON_ARRAY(
+        	    'ROWS',
+            	'COUNT'
+    	    ),
+	        'required', false
+		),
+		JSON_OBJECT(
+			'name', 'threshold',
+        	'displayName', 'Threshold',
+        	'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
+        	'dataType', 'NUMBER',
+        	'required', false
+		)
+	)	
+)
+WHERE name = 'tableCustomSQLQuery';
+
+-- Modify migrations for service connection of airflow to move password under authType if 
+-- Connection Type as Mysql or Postgres
+
+UPDATE pipeline_service_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.connection.password'),
+    '$.connection.config.connection.authType',
+    JSON_OBJECT(),
+    '$.connection.config.connection.authType.password',
+    JSON_EXTRACT(json, '$.connection.config.connection.password'))
+where serviceType = 'Airflow' 
+AND JSON_EXTRACT(json, '$.connection.config.connection.type') in ('Postgres', 'Mysql')
+AND JSON_EXTRACT(json, '$.connection.config.connection.password') IS NOT NULL;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql

@@ -0,0 +1,47 @@
+-- Rename includeTempTables with includeTransTables
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(
+    JSON_SET(
+        json,
+        '$.connection.config.includeTransientTables',
+        JSON_EXTRACT(json, '$.connection.config.includeTempTables')
+    ),
+    '$.connection.config.includeTempTables'
+)
+WHERE serviceType in ('Snowflake') AND JSON_EXTRACT(json, '$.connection.config.includeTempTables') IS NOT NULL;
+
+UPDATE dbservice_entity
+SET json = JSON_REPLACE(json, '$.connection.config.scheme', 'hive')
+WHERE JSON_EXTRACT(json, '$.connection.config.scheme') IN ('impala', 'impala4')
+AND serviceType = 'Hive';
+
+-- remove the dataModel references from Data Models
+UPDATE dashboard_data_model_entity
+SET json = JSON_REMOVE(json, '$.dataModels');
+
+-- migrate ingestAllDatabases in mssql
+UPDATE dbservice_entity de2 
+SET json = JSON_REPLACE(
+    JSON_INSERT(json, 
+      '$.connection.config.database', 
+      (select JSON_EXTRACT(json, '$.name') 
+        from database_entity de 
+        where id = (select er.toId 
+            from entity_relationship er 
+            where er.fromId = de2.id 
+              and er.toEntity = 'database' 
+            LIMIT 1
+          ))
+    ), '$.connection.config.ingestAllDatabases', 
+    true
+  ) 
+where de2.serviceType = 'Mssql' 
+  and JSON_EXTRACT(json, '$.connection.config.database') is NULL;
+
+-- remove keyfile from clickhouse
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.keyfile')
+WHERE serviceType = 'Clickhouse';
+
+-- Clean old test connections
+TRUNCATE automations_workflow;

bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v015__update_server_change_log.sql

@@ -0,0 +1,2 @@
+-- This column is already created in v000__create_server_change_log.sql
+-- ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics JSON;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v000__create_server_change_log.sql

@@ -0,0 +1,18 @@
+-- Create tables for tracking server migrations
+-- This migration runs before all other migrations to ensure migration tracking tables exist
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank SERIAL,
+    version VARCHAR(256) PRIMARY KEY,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    metrics JSONB
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v001__create_db_connection_info.sql

@@ -0,0 +1,430 @@
+CREATE OR REPLACE FUNCTION to_tz_timestamp(text) RETURNS TIMESTAMP WITH TIME ZONE AS
+$$
+select to_timestamp($1, '%Y-%m-%dT%T.%fZ')::timestamptz;
+$$
+    LANGUAGE sql immutable;
+
+--
+-- Table that captures all the relationships between entities
+--
+CREATE TABLE IF NOT EXISTS entity_relationship (
+    fromId VARCHAR(36) NOT NULL,                -- ID of the from entity
+    toId VARCHAR(36) NOT NULL,                  -- ID of the to entity
+    fromEntity VARCHAR(256) NOT NULL,           -- Type name of the from entity
+    toEntity VARCHAR(256) NOT NULL,             -- Type name of to entity
+    relation SMALLINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSONB,                                 -- JSON payload with additional information
+    deleted BOOLEAN NOT NULL DEFAULT FALSE,
+    PRIMARY KEY (fromId, toId, relation)
+);
+
+CREATE INDEX IF NOT EXISTS entity_relationship_edge_index ON entity_relationship(fromId, toId, relation);
+CREATE INDEX IF NOT EXISTS entity_relationship_from_index ON entity_relationship(fromId, relation);
+CREATE INDEX IF NOT EXISTS entity_relationship_to_index ON entity_relationship(toId, relation);
+
+--
+-- Table that captures all the relationships between field of an entity to a field of another entity
+-- Example - table1.column1 (fromFQN of type table.columns.column) is joined with(relation)
+-- table2.column8 (toFQN of type table.columns.column)
+--
+CREATE TABLE IF NOT EXISTS field_relationship (
+    fromFQN VARCHAR(256) NOT NULL,              -- Fully qualified name of entity or field
+    toFQN VARCHAR(256) NOT NULL,                -- Fully qualified name of entity or field
+    fromType VARCHAR(256) NOT NULL,             -- Fully qualified type of entity or field
+    toType VARCHAR(256) NOT NULL,               -- Fully qualified type of entity or field
+    relation SMALLINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSONB,                                 -- JSON payload with additional information
+    PRIMARY KEY (fromFQN, toFQN, relation)
+);
+
+CREATE INDEX IF NOT EXISTS field_relationship_from_index ON field_relationship(fromFQN, relation);
+CREATE INDEX IF NOT EXISTS field_relationship_to_index ON field_relationship(toFQN, relation);
+
+--
+-- Used for storing additional metadata for an entity
+--
+CREATE TABLE IF NOT EXISTS entity_extension (
+    id VARCHAR(36) NOT NULL,                    -- ID of the from entity
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSONB NOT NULL,
+    PRIMARY KEY (id, extension)
+);
+
+--
+-- Service entities
+--
+CREATE TABLE IF NOT EXISTS dbservice_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS messaging_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS storage_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Data entities
+--
+CREATE TABLE IF NOT EXISTS database_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS database_schema_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt') :: bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted') :: boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS table_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS metric_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS report_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS ml_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS topic_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS chart_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS location_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Feed related tables
+--
+CREATE TABLE IF NOT EXISTS thread_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityId') STORED NOT NULL,
+    entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'about') STORED NOT NULL,
+    assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> 'addressedTo') STORED,
+    json JSONB NOT NULL,
+    createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'threadTs')::bigint) STORED NOT NULL,
+    createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    resolved BOOLEAN GENERATED ALWAYS AS ((json ->> 'resolved')::boolean) STORED,
+    PRIMARY KEY (id)
+);
+
+--
+-- Policies related tables
+--
+CREATE TABLE IF NOT EXISTS policy_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Ingestion related tables
+--
+CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    timestamp BIGINT,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- User, Team, and bots
+--
+CREATE TABLE IF NOT EXISTS team_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS user_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    email VARCHAR(256) GENERATED ALWAYS AS (json ->> 'email') STORED NOT NULL,
+    deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> 'deactivated') STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS bot_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS role_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    defaultRole BOOLEAN GENERATED ALWAYS AS ((json ->> 'defaultRole')::boolean) STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Usage table where usage for all the entities is captured
+--
+CREATE TABLE IF NOT EXISTS entity_usage (
+    id VARCHAR(36) NOT NULL,         -- Unique id of the entity
+    entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
+    usageDate DATE,                  -- date corresponding to the usage
+    count1 INT,                      -- total daily count of use on usageDate
+    count7 INT,                      -- rolling count of last 7 days going back from usageDate
+    count30 INT,                     -- rolling count of last 30 days going back from usageDate
+    percentile1 INT,                 -- percentile rank with in same entity for given usage date
+    percentile7 INT,                 -- percentile rank with in same entity for last 7 days of usage
+    percentile30 INT,                -- percentile rank with in same entity for last 30 days of usage
+    UNIQUE (usageDate, id)
+);
+
+--
+-- Tag related tables
+--
+CREATE TABLE IF NOT EXISTS tag_category (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL, -- JSON stores category information and does not store children
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name) -- Unique tag category name
+);
+
+CREATE TABLE IF NOT EXISTS tag (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL, -- JSON stores all tag attributes and does not store children
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS tag_usage (
+    source SMALLINT NOT NULL,           -- Source of the tag label
+    tagFQN VARCHAR(256) NOT NULL,       -- Fully qualified name of the tag
+    targetFQN VARCHAR(256) NOT NULL,    -- Fully qualified name of the entity instance or corresponding field
+    labelType SMALLINT NOT NULL,        -- Type of tagging: manual, automated, propagated, derived
+    state SMALLINT NOT NULL,            -- State of tagging: suggested or confirmed
+    UNIQUE (source, tagFQN, targetFQN)
+);
+
+CREATE TABLE IF NOT EXISTS change_event (
+    eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
+    userName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'userName') STORED NOT NULL,
+    eventTime BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
+    json JSONB NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS change_event_event_type_index ON change_event(eventType);
+CREATE INDEX IF NOT EXISTS change_event_entity_type_index ON change_event(entityType);
+CREATE INDEX IF NOT EXISTS change_event_event_time_index ON change_event(eventTime);
+
+CREATE TABLE IF NOT EXISTS webhook_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+    -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
+);
+
+CREATE TABLE IF NOT EXISTS glossary_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS glossary_term_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v002__create_db_connection_info.sql

@@ -0,0 +1,78 @@
+CREATE TABLE IF NOT EXISTS type_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+ALTER TABLE webhook_entity
+ADD status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED NOT NULL,
+DROP COLUMN deleted;
+
+DROP INDEX entity_relationship_edge_index;
+
+CREATE TABLE IF NOT EXISTS mlmodel_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE thread_entity
+SET json = jsonb_set(json, '{type}', '"Conversation"', true);
+
+update thread_entity
+SET json = jsonb_set(json, '{reactions}', '[]'::jsonb, true);
+
+ALTER TABLE thread_entity
+    ADD type VARCHAR(64) GENERATED ALWAYS AS (json ->> 'type') STORED NOT NULL,
+    ADD taskId INT GENERATED ALWAYS AS ((json#>'{task,id}')::integer) STORED,
+    ADD taskStatus VARCHAR(64) GENERATED ALWAYS AS (json#>>'{task,status}') STORED,
+    ADD taskAssignees JSONB GENERATED ALWAYS AS (json#>'{task,assignees}') STORED,
+    ADD CONSTRAINT task_id_constraint UNIQUE(taskId);
+
+CREATE INDEX IF NOT EXISTS thread_entity_type_index ON thread_entity(type);
+CREATE INDEX IF NOT EXISTS thread_entity_task_assignees_index ON thread_entity(taskAssignees);
+CREATE INDEX IF NOT EXISTS thread_entity_task_status_index ON thread_entity(taskStatus);
+CREATE INDEX IF NOT EXISTS thread_entity_created_by_index ON thread_entity(createdBy);
+CREATE INDEX IF NOT EXISTS thread_entity_updated_at_index ON thread_entity(updatedAt);
+
+CREATE TABLE task_sequence (id SERIAL PRIMARY KEY, dummy varchar(1));
+INSERT INTO task_sequence (dummy) VALUES (0) RETURNING id;
+
+DELETE from ingestion_pipeline_entity where 1=1;
+DELETE FROM pipeline_service_entity WHERE 1=1;
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,databaseSchema}', json#>'{connection,config,database}')
+where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle')
+  and json#>'{connection,config,database}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle');
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}' #- '{connection,config,username}' #- '{connection,config,projectId}' #- '{connection,config,enablePolicyTagImport}'
+WHERE serviceType = 'BigQuery';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+WHERE serviceType in ('Athena','Databricks');
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}' #- '{connection,config,pipelineServiceName}'
+WHERE serviceType = 'Glue';
+
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,dbServiceName}'
+WHERE serviceType in ('Metabase','Superset','Tableau');

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v003__create_db_connection_info.sql

@@ -0,0 +1,31 @@
+DELETE from ingestion_pipeline_entity where 1=1;
+DELETE from entity_relationship where toEntity = 'ingestionPipeline';
+
+-- 0.10 had empty FQNs for services, users and teams
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE messaging_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE storage_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE team_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE user_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v004__create_db_connection_info.sql

@@ -0,0 +1,113 @@
+UPDATE team_entity
+SET json = JSONB_SET(json, '{teamType}', '"Group"', true);
+
+ALTER TABLE team_entity
+ADD teamType VARCHAR(64) GENERATED ALWAYS AS (json ->> 'teamType') STORED NOT NULL;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+where serviceType = 'DynamoDB';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,connectionOptions}'
+where serviceType = 'DeltaLake';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}'
+where serviceType = 'DeltaLake';
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{connection,config,clientId}', json#>'{connection,config,username}')
+WHERE serviceType = 'Looker'
+  and json#>'{connection,config,username}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{connection,config,clientSecret}', json#>'{connection,config,password}')
+WHERE serviceType = 'Looker'
+  and json#>'{connection,config,password}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}' #- '{connection,config,env}'
+WHERE serviceType = 'Looker';
+
+CREATE TABLE IF NOT EXISTS test_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_suite (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_case (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    entityFQN VARCHAR (712) GENERATED ALWAYS AS (json ->> 'entityFQN') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE webhook_entity
+SET json = JSONB_SET(json::jsonb, '{webhookType}', '"generic"', true);
+
+ALTER TABLE webhook_entity
+    ADD webhookType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'webhookType') STORED NOT NULL;
+
+CREATE TABLE IF NOT EXISTS entity_extension_time_series (
+    entityFQN VARCHAR(1024) NOT NULL,           -- Entity FQN, we can refer to tables and columns
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSONB NOT NULL,
+    timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
+);
+
+
+ALTER TABLE thread_entity
+    ADD announcementStart BIGINT GENERATED ALWAYS AS ((json#>'{announcement,startTime}')::bigint) STORED,
+    ADD announcementEnd BIGINT GENERATED ALWAYS AS ((json#>'{announcement,endTime}')::bigint) STORED;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,databaseSchema}' #- '{connection,config,oracleServiceName}'
+WHERE serviceType = 'Oracle';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,hostPort}'
+WHERE serviceType = 'Athena';
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}'
+WHERE serviceType in ('Databricks');
+
+CREATE TABLE IF NOT EXISTS openmetadata_settings (
+    id SERIAL NOT NULL ,
+    configType VARCHAR(36) NOT NULL,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id, configType),
+    UNIQUE(configType)
+);
+
+DELETE FROM entity_extension
+WHERE jsonSchema IN ('tableProfile', 'columnTest', 'tableTest');
+
+DELETE FROM ingestion_pipeline_entity 
+WHERE json_extract_path_text("json", 'pipelineType') = 'profiler';
+
+DELETE FROM role_entity;
+DELETE FROM policy_entity;
+DELETE FROM field_relationship WHERE fromType IN ('role', 'policy') OR toType IN ('role', 'policy');
+DELETE FROM entity_relationship WHERE fromEntity IN ('role', 'policy') OR toEntity IN ('role', 'policy');
+ALTER TABLE role_entity DROP COLUMN defaultRole;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v005__create_db_connection_info.sql

@@ -0,0 +1,347 @@
+DELETE FROM entity_relationship
+WHERE toEntity = 'ingestionPipeline'
+AND toId NOT IN (
+	SELECT DISTINCT id 
+	FROM ingestion_pipeline_entity
+);
+
+CREATE TABLE IF NOT EXISTS user_tokens (
+    token VARCHAR(36) GENERATED ALWAYS AS (json ->> 'token') STORED NOT NULL,
+    userId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'userId') STORED NOT NULL,
+    tokenType VARCHAR(50) GENERATED ALWAYS AS (json ->> 'tokenType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    expiryDate BIGINT GENERATED ALWAYS AS ((json ->> 'expiryDate')::bigint) STORED NOT NULL,
+    PRIMARY KEY (token)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(
+        json,
+        '{connection,config,metastoreConnection}',
+        jsonb_build_object('metastoreHostPort', json#>'{connection,config,metastoreHostPort}')
+    )
+WHERE serviceType = 'DeltaLake'
+  AND json#>'{connection,config,metastoreHostPort}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,metastoreHostPort}'
+WHERE serviceType = 'DeltaLake';
+
+UPDATE dbservice_entity
+SET json = jsonb_set(
+        json,
+        '{connection,config,metastoreConnection}',
+        jsonb_build_object('metastoreFilePath', json#>'{connection,config,metastoreFilePath}')
+    )
+WHERE serviceType = 'DeltaLake'
+  AND json#>'{connection,config,metastoreFilePath}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,metastoreFilePath}'
+WHERE serviceType = 'DeltaLake';
+
+ALTER TABLE test_definition 
+ADD supported_data_types JSONB GENERATED ALWAYS AS (json -> 'supportedDataTypes') STORED;
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnCountToEqual"',
+  false
+)
+WHERE json->>'name' = 'TableColumnCountToEqual';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnCountToEqual"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnCountToEqual';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnToMatchSet"',
+  false
+)
+WHERE json->>'name' = 'TableColumnToMatchSet';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnToMatchSet"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnToMatchSet';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnNameToExist"',
+  false
+)
+WHERE json->>'name' = 'TableColumnNameToExist';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnNameToExist"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnNameToExist';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableRowCountToBeBetween"',
+  false
+)
+WHERE json->>'name' = 'TableRowCountToBeBetween';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableRowCountToBeBetween"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableRowCountToBeBetween';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnCountToBeBetween"',
+  false
+)
+WHERE json->>'name' = 'TableColumnCountToBeBetween';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnCountToBeBetween"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnCountToBeBetween';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"columnValuesToBeInSet"',
+  false
+)
+WHERE json->>'name' = 'ColumnValuesToBeInSet';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"columnValuesToBeInSet"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'ColumnValuesToBeInSet';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableRowCountToEqual"',
+  false
+)
+WHERE json->>'name' = 'TableRowCountToEqual';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableRowCountToEqual"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableRowCountToEqual';
+
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMeanToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "ARRAY", "SET"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMaxToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMedianToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMinToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueLengthsToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesMissingCount'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesSumToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueStdDevToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeNotInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeNotNull'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeUnique'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToNotMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE pipeline_service_entity
+SET json = json::jsonb #- '{connection,config,dbConnection}'
+WHERE serviceType = 'Dagster';
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(
+      jsonb_set(
+        json,
+        '{serviceType}',
+        '"GluePipeline"',
+        true
+      ),
+      '{connection,config,type}',
+      '"GluePipeline"',
+      true
+  )
+WHERE serviceType = 'Glue';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v006__create_db_connection_info.sql

@@ -0,0 +1,155 @@
+--
+-- Upgrade changes for 0.13
+--
+
+
+ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQN TYPE varchar(768);
+CREATE INDEX IF NOT EXISTS entity_extension_time_series_entity_fqn_index ON entity_extension_time_series(entityFQN);
+
+CREATE TABLE IF NOT EXISTS web_analytic_event (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
+
+UPDATE bot_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
+
+UPDATE bot_entity
+SET json = json::jsonb #- '{botType}';
+
+CREATE TABLE IF NOT EXISTS data_insight_chart (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'dataIndexType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
+
+UPDATE role_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE name in ('DataConsumer', 'DataSteward');
+
+UPDATE policy_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE fullyQualifiedName in ('DataConsumerPolicy', 'DataStewardPolicy', 'OrganizationPolicy', 'TeamOnlyPolicy');
+
+UPDATE tag_category
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE fullyQualifiedName in ('PersonalData.Personal', 'PersonalData.SpecialCategory',
+'PII.None', 'PII.NonSensitive', 'PII.Sensitive',
+'Tier.Tier1', 'Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5');
+
+UPDATE pipeline_service_entity 
+SET json = JSONB_SET(json::jsonb,'{connection,config}',json::jsonb #>'{connection,config}' || jsonb_build_object('configSource',jsonb_build_object('hostPort',json #>'{connection,config,hostPort}')), true)
+where servicetype = 'Dagster';
+
+UPDATE pipeline_service_entity 
+SET json = json::jsonb #- '{connection,config,hostPort}' #- '{connection,config,numberOfStatus}'
+where servicetype = 'Dagster';
+
+-- Remove categoryType
+UPDATE tag_category
+SET json = json::jsonb #- '{categoryType}';
+
+-- set mutuallyExclusive flag
+UPDATE tag_category
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
+
+UPDATE tag_category
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'true'::jsonb, true)
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
+
+-- Merge mutually exclusive table when multiple tag labels are used
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier1' AND
+t1.tagFQN in ('Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier2' AND
+t1.tagFQN in ('Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier3' AND
+t1.tagFQN in ('Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier4' AND
+t1.tagFQN in ('Tier.Tier5'));
+
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PII.Sensitive' AND
+t1.tagFQN in ('PII.NonSensitive', 'PII.None')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PII.NonSensitive' AND
+t1.tagFQN in ('PII.None'));
+
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PersonalData.Personal' AND
+t1.tagFQN in ('PersonalData.SpecialCategory'));
+
+CREATE TABLE IF NOT EXISTS kpi_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS metadata_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- We are starting to store the current deployed flag. Let's mark it as false by default
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{deployed}';
+
+UPDATE ingestion_pipeline_entity
+SET json = jsonb_set(json::jsonb, '{deployed}', 'true'::jsonb, true);
+
+-- We removed the supportsMetadataExtraction field in the `OpenMetadataConnection` object being used in IngestionPipelines
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{openMetadataServerConnection,supportsMetadataExtraction}';
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v007__create_db_connection_info.sql

@@ -0,0 +1,63 @@
+-- Remove markDeletedTablesFromFilterOnly 
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,markDeletedTablesFromFilterOnly}';
+
+UPDATE data_insight_chart
+SET json = jsonb_set(
+        json,
+        '{dimensions}',
+        '[{"name":"entityFqn","chartDataType":"STRING"},{"name":"entityType","chartDataType":"STRING"},{"name":"owner","chartDataType":"STRING"},{"name":"entityHref","chartDataType":"STRING"}]'
+)
+WHERE name = 'mostViewedEntities';
+
+DROP TABLE webhook_entity;
+
+CREATE TABLE IF NOT EXISTS alert_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS alert_action_def (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    alertActionType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'alertActionType') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,database}', json#>'{connection,config,databaseSchema}')
+where serviceType in ('Db2')
+  and json#>'{connection,config,databaseSchema}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,databaseSchema}'
+where serviceType in ('Db2');
+
+DELETE from openmetadata_settings where configType = 'activityFeedFilterSetting';
+
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource}';
+
+UPDATE pipeline_service_entity 
+SET json = jsonb_set(jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,token}', json#> '{connection,config,configSource,token}', true) ,'{connection,config,host}', json #> '{connection,config,configSource,host}' , true)
+WHERE serviceType = 'Dagster' and json #>'{connection,config,configSource,host}' is not null;
+
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,host}', json#> '{connection,config,configSource,hostPort}', true)
+WHERE servicetype = 'Dagster' and json #>'{connection,config,configSource,hostPort}' is not null;
+
+UPDATE topic_entity
+SET json = jsonb_set(json::jsonb #- '{schemaText}', '{messageSchema}', jsonb_build_object('schemaText', json#>'{schemaText}'), true)
+WHERE json #> '{schemaText}' IS NOT NULL;
+
+UPDATE topic_entity
+SET json = jsonb_set(json::jsonb #- '{schemaType}', '{messageSchema,schemaType}', json#> '{schemaType}', true)
+WHERE json #> '{schemaType}' IS NOT NULL;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v008__create_db_connection_info.sql

@@ -0,0 +1,53 @@
+ALTER TABLE tag_category
+RENAME TO classification;
+
+-- Rename tagCategoryName in BigQuery for classificationName
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,classificationName}', json#>'{connection,config,tagCategoryName}')
+where serviceType in ('BigQuery')
+  and json#>'{connection,config,tagCategoryName}' is not null;
+
+-- Delete supportsUsageExtraction from vertica
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsUsageExtraction}'
+WHERE serviceType = 'Vertica';
+
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtUpdateDescriptions}'
+WHERE json#>>'{sourceConfig,config,type}' = 'DBT';
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "TIMESTAMP", "TIMESTAMPZ","DATETIME", "DATE"]',
+  false
+)
+WHERE json->>'name' = 'columnValuesToBeBetween';
+
+UPDATE pipeline_entity
+SET json = jsonb_set(
+        json,
+        '{name}',
+        to_jsonb(replace(json ->> 'name',':',''))
+    )
+WHERE json ->> 'serviceType' = 'Dagster';
+
+UPDATE pipeline_entity
+SET json = jsonb_set(
+        json,
+        '{fullyQualifiedName}',
+        to_jsonb(replace(json ->> 'fullyQualifiedName',':',''))
+    )
+WHERE json ->> 'serviceType' = 'Dagster';
+
+UPDATE dashboard_service_entity  
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}' #- '{provider}'|| 
+jsonb_build_object('connection',jsonb_build_object(
+'username',json #>'{connection,config,username}',
+'password',json #>'{connection,config,password}',
+'provider',json #>'{connection,config,provider}'
+)), true)
+where servicetype = 'Superset';
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v009__create_db_connection_info.sql

@@ -0,0 +1,231 @@
+-- Unique constraint for user email address
+ALTER TABLE user_entity
+ADD UNIQUE (email);
+
+
+-- Remove classificationName in BigQuery
+UPDATE dbservice_entity SET json = json #- '{connection,config,classificationName}' where serviceType in ('BigQuery');
+
+-- migrate ingestAllDatabases in postgres 
+UPDATE dbservice_entity de2 
+SET json = JSONB_SET(
+    json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'|| 
+    jsonb_build_object('database',
+    (SELECT json->>'name' 
+        FROM database_entity de 
+        WHERE id = (SELECT er.toId 
+                    FROM entity_relationship er 
+                    WHERE er.fromId = de2.id 
+                    AND er.toEntity = 'database' 
+                    LIMIT 1)
+    )
+    )), 
+    '{connection,config,ingestAllDatabases}', 
+    'true'::jsonb
+)
+WHERE de2.serviceType = 'Postgres' 
+AND json->>'{connection,config,database}' IS NULL;
+
+CREATE TABLE IF NOT EXISTS storage_container_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS test_connection_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS automations_workflow (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'workflowType') STORED NOT NULL,
+    status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- Do not store OM server connection, we'll set it dynamically on the resource
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{openMetadataServerConnection}';
+
+CREATE TABLE IF NOT EXISTS query_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)  
+);
+
+CREATE TABLE IF NOT EXISTS temp_query_migration (
+    tableId VARCHAR(36) NOT NULL,
+    queryId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    queryName VARCHAR(255) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL
+);
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+INSERT INTO temp_query_migration(tableId,json)
+SELECT id,json_build_object('id',gen_random_uuid(),'query',query,'users',users,'checksum',checksum,'duration',duration,'name',checksum,'updatedAt',
+floor(EXTRACT(EPOCH FROM NOW())),'updatedBy','admin','deleted',false) AS json FROM entity_extension AS ee , jsonb_to_recordset(ee.json) AS x (query varchar,users json,
+checksum varchar,name varchar, duration decimal,queryDate varchar)
+WHERE ee.extension = 'table.tableQueries';
+
+INSERT INTO query_entity (json)
+SELECT value
+FROM (
+  SELECT jsonb_object_agg(queryName, json) AS json_data FROM ( SELECT DISTINCT queryName, json FROM temp_query_migration) subquery
+) cte, jsonb_each(cte.json_data)
+ON CONFLICT (name) DO UPDATE SET json = EXCLUDED.json;
+
+INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation)
+SELECT tmq.tableId, qe.id, 'table', 'query', 5
+FROM temp_query_migration tmq
+JOIN query_entity qe ON qe.name = tmq.queryName;
+
+DELETE FROM entity_extension WHERE id in
+(SELECT DISTINCT tableId FROM temp_query_migration) AND extension = 'table.tableQueries';
+
+DROP TABLE temp_query_migration;
+
+-- remove the audience if it was wrongfully sent from the UI after editing the OM service
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{connection,config,securityConfig,audience}'
+WHERE name = 'OpenMetadata'
+    AND json#>'{connection,config,authProvider}' IS NOT NULL
+    AND json -> 'connection' -> 'config' ->> 'authProvider' != 'google';
+
+ALTER TABLE user_tokens ALTER COLUMN expiryDate DROP NOT NULL;
+
+CREATE TABLE IF NOT EXISTS event_subscription_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+drop table if exists alert_action_def;
+drop table if exists alert_entity;
+DELETE from entity_relationship where  fromEntity = 'alert' and toEntity = 'alertAction';
+
+-- create data model table
+CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,database}', '{connection,config,databaseName}', json#> '{connection,config,database}', true)
+WHERE servicetype = 'Druid' and json #>'{connection,config,database}' is not null;
+
+-- We were using the same jsonSchema for Pipeline Services and Ingestion Pipeline status
+-- Also, we relied on the extension to store the run id
+UPDATE entity_extension_time_series
+SET jsonSchema = 'ingestionPipelineStatus', extension = 'ingestionPipeline.pipelineStatus'
+WHERE jsonSchema = 'pipelineStatus' AND extension <> 'pipeline.PipelineStatus';
+
+-- We are refactoring the storage service with containers. We'll remove the locations
+DROP TABLE location_entity;
+DELETE FROM entity_relationship WHERE fromEntity='location' OR toEntity='location';
+TRUNCATE TABLE storage_service_entity;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,storageServiceName}'
+WHERE servicetype = 'Glue';
+
+UPDATE chart_entity
+SET json = json::jsonb #- '{tables}';
+
+-- Updating the tableau authentication fields
+UPDATE dashboard_service_entity
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}'|| 
+jsonb_build_object('authType',jsonb_build_object(
+'username',json #>'{connection,config,username}',
+'password',json #>'{connection,config,password}'
+)), true)
+where servicetype = 'Tableau'
+and json#>'{connection,config,password}' is not null
+and json#>'{connection,config,username}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{personalAccessTokenName}' #- '{personalAccessTokenSecret}'|| 
+jsonb_build_object('authType',jsonb_build_object(
+'personalAccessTokenName',json #>'{connection,config,personalAccessTokenName}',
+'personalAccessTokenSecret',json #>'{connection,config,personalAccessTokenSecret}'
+)), true)
+where servicetype = 'Tableau'
+and json#>'{connection,config,personalAccessTokenName}' is not null
+and json#>'{connection,config,personalAccessTokenSecret}' is not null;
+
+-- Removed property from metadataService.json
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{allowServiceCreation}'
+WHERE serviceType in ('Amundsen', 'Atlas', 'MetadataES', 'OpenMetadata');
+
+UPDATE metadata_service_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"')
+WHERE name = 'OpenMetadata';
+
+-- Fix Glue sample data endpoint URL to be a correct URI
+UPDATE dbservice_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,awsConfig,endPointURL}', '"https://glue.region_name.amazonaws.com/"')
+WHERE serviceType = 'Glue'
+  AND json#>'{connection,config,awsConfig,endPointURL}' = '"https://glue.<region_name>.amazonaws.com/"';
+
+-- Delete connectionOptions from superset
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,connectionOptions}'
+WHERE serviceType = 'Superset';
+
+-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,partitionQueryDuration}' #- '{connection,config,partitionQuery}' #- '{connection,config,partitionField}'
+WHERE serviceType = 'BigQuery';
+
+-- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsQueryComment}' #- '{connection,config,scheme}' #- '{connection,config,hostPort}' #- '{connection,config,supportsProfiler}'
+WHERE serviceType = 'Salesforce';
+
+-- Delete supportsProfiler from DynamoDB
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}'
+WHERE serviceType = 'DynamoDB';
+
+-- Update TagLabels source from 'Tag' to 'Classification' after #10486
+UPDATE table_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
+UPDATE ml_model_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
+
+-- Delete uriString from Mssql
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,uriString}'
+WHERE serviceType = 'Mssql';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v010__create_db_connection_info.sql

@@ -0,0 +1,17 @@
+-- Updating the value of SASL Mechanism for Kafka and Redpanda connections
+UPDATE messaging_service_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,saslMechanism}', '"PLAIN"')
+WHERE (servicetype = 'Kafka' OR serviceType = 'Redpanda')
+  AND json#>'{connection,config,saslMechanism}' IS NOT NULL
+  AND json#>'{connection,config,saslMechanism}' NOT IN ('"GSSAPI"', '"PLAIN"', '"SCRAM-SHA-256"', '"SCRAM-SHA-512"', '"OAUTHBEARER"');
+
+-- Remove the Subscriptions
+DELETE FROM event_subscription_entity;
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove the ibmi scheme from Db2 replace it with db2+ibm_db
+UPDATE dbservice_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,scheme}', '"db2+ibm_db"')
+WHERE serviceType  = 'Db2';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v011__create_db_connection_info.sql

@@ -0,0 +1,4 @@
+-- Update the tableau data model enum
+UPDATE dashboard_data_model_entity
+SET json = JSONB_SET(json::jsonb, '{dataModelType}', '"TableauDataModel"')
+WHERE json#>'{dataModelType}' = '"TableauSheet"';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v012__create_db_connection_info.sql

@@ -0,0 +1,7 @@
+-- use FQN instead of name for Test Connection Definition
+ALTER TABLE test_connection_definition
+ADD fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+DROP COLUMN name;
+
+-- Since we are not deleting the test connection defs anymore, clean it up
+TRUNCATE test_connection_definition;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v013__create_db_connection_info.sql

@@ -0,0 +1,193 @@
+-- we are not using the secretsManagerCredentials
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{openMetadataServerConnection.secretsManagerCredentials}'
+where name = 'OpenMetadata';
+
+-- Rename githubCredentials to gitCredentials
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,githubCredentials}', '{connection,config,gitCredentials}', json#>'{connection,config,githubCredentials}')
+    where serviceType = 'Looker'
+  and json#>'{connection,config,githubCredentials}' is not null;
+
+-- Rename gcsConfig in BigQuery to gcpConfig
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,credentials,gcsConfig}', '{connection,config,credentials,gcpConfig}',
+json#>'{connection,config,credentials,gcsConfig}')
+where serviceType in ('BigQuery')
+  and json#>'{connection,config,credentials,gcsConfig}' is not null;
+
+-- Rename gcsConfig in Datalake to gcpConfig
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,configSource,securityConfig,gcsConfig}', '{connection,config,configSource,securityConfig,gcpConfig}',
+json#>'{connection,config,configSource,securityConfig,gcsConfig}')
+where serviceType in ('Datalake')
+  and json#>'{connection,config,configSource,securityConfig,gcsConfig}' is not null;
+
+-- Rename gcsConfig in dbt to gcpConfig
+UPDATE ingestion_pipeline_entity
+SET json = jsonb_set(json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}', '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}', (json#>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}')::jsonb)
+WHERE json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig}' is not null and json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}' is not null;
+
+
+-- Rename dashboardUrl in dashboard_entity to sourceUrl
+UPDATE dashboard_entity
+SET json = jsonb_set(json::jsonb #- '{dashboardUrl}' , '{sourceUrl}',
+json#>'{dashboardUrl}')
+where json#>'{dashboardUrl}' is not null;
+
+-- Rename chartUrl in chart_entity to sourceUr
+UPDATE chart_entity
+SET json = jsonb_set(json::jsonb #- '{chartUrl}' , '{sourceUrl}',
+json#>'{chartUrl}')
+where json#>'{chartUrl}' is not null;
+
+-- Rename pipelineUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity
+SET json = jsonb_set(json::jsonb #- '{pipelineUrl}' , '{sourceUrl}',
+json#>'{pipelineUrl}')
+where json#>'{pipelineUrl}' is not null;
+
+
+-- Rename taskUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity
+SET json = jsonb_set(
+    json::jsonb - 'tasks',
+    '{tasks}',
+    (
+        SELECT jsonb_agg(
+            jsonb_build_object(
+                'name', t ->> 'name',
+                'sourceUrl', t ->> 'taskUrl',
+                'taskType', t ->> 'taskType',
+                'description', t ->> 'description',
+                'displayName', t ->> 'displayName',
+                'fullyQualifiedName', t ->> 'fullyQualifiedName',
+                'downstreamTasks', (t -> 'downstreamTasks')::jsonb,
+                'tags', (t ->> 'tags')::jsonb,
+                'endDate', t ->> 'endDate',
+                'startDate', t ->> 'startDate',
+                'taskSQL', t ->> 'taskSQL'
+            )
+        )
+        FROM jsonb_array_elements(json->'tasks') AS t
+    )
+)
+where json#>'{tasks}' is not null;
+
+
+-- Modify migrations for service connection of postgres and mysql to move password under authType
+UPDATE dbservice_entity
+SET json =  jsonb_set(
+json #-'{connection,config,password}',
+'{connection,config,authType}',
+jsonb_build_object('password',json#>'{connection,config,password}')
+)
+WHERE serviceType IN ('Postgres', 'Mysql')
+  and json#>'{connection,config,password}' is not null;
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove sourceUrl in pipeline_entity from DatabricksPipeline & Fivetran
+UPDATE pipeline_entity
+SET json = json::jsonb #- '{sourceUrl}'
+where json #> '{serviceType}' in ('"DatabricksPipeline"','"Fivetran"');
+
+
+-- Remove sourceUrl in dashboard_entity from Mode
+UPDATE dashboard_entity
+SET json = json::jsonb #- '{sourceUrl}'
+where json #> '{serviceType}' in ('"Mode"');
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank SERIAL,
+    version VARCHAR(256) PRIMARY KEY,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Update test definition parameterValues
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueMeanToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueMedianToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueStdDevToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minLength", "dataType": "INT", "required": false, "description": "The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length", "displayName": "Min", "optionValues": []}, {"name": "maxLength", "dataType": "INT", "required": false, "description": "The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueLengthsToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValue", "dataType": "INT", "required": false, "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValuesToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "columnNames", "dataType": "STRING", "required": true, "description": "Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string", "displayName": "Column Names", "optionValues": []}, {"name": "ordered", "dataType": "BOOLEAN", "required": false, "description": "Whether or not to considered the order of the list when performing the match check", "displayName": "Ordered", "optionValues": []}]'
+        )
+where name = 'tableColumnToMatchSet';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'tableRowCountToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name":"sqlExpression","displayName":"SQL Expression","description":"SQL expression to run against the table","dataType":"STRING","required":"true"},{"name":"strategy","displayName":"Strategy","description":"Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to ROWS)","dataType":"ARRAY","optionValues":["ROWS","COUNT"],"required":false},{"name":"threshold","displayName":"Threshold","description":"Threshold to use to determine if the test passes or fails (defaults to 0).","dataType":"NUMBER","required":false}]'
+        )
+where name = 'tableCustomSQLQuery';
+
+-- Modify migrations for service connection of airflow to move password under authType if 
+-- Connection Type as Mysql or Postgres
+
+UPDATE pipeline_service_entity
+SET json =  jsonb_set(
+json #-'{connection,config,connection,password}',
+'{connection,config,connection,authType}',
+jsonb_build_object('password',json#>'{connection,config,connection,password}')
+)
+WHERE serviceType = 'Airflow'
+and json#>'{connection,config,connection,type}' IN ('"Mysql"', '"Postgres"')
+and json#>'{connection,config,connection,password}' is not null;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v014__create_db_connection_info.sql

@@ -0,0 +1,43 @@
+-- Rename includeTempTables in snowflake to includeTransientTables
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,includeTempTables}', '{connection,config,includeTransientTables}',
+json#>'{connection,config,includeTempTables}')
+where serviceType in ('Snowflake') and json#>'{connection,config,includeTempTables}' is not null ;
+
+
+update dbservice_entity
+set json = jsonb_set(json::jsonb, '{connection,config,scheme}', '"hive"')
+where json#>>'{connection,config,scheme}' in ('impala', 'impala4')
+and serviceType = 'Hive';
+
+-- remove the dataModel references from Data Models
+UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}';
+
+-- migrate ingestAllDatabases in mssql
+UPDATE dbservice_entity de2 
+SET json = JSONB_SET(
+    json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'|| 
+    jsonb_build_object('database',
+    (SELECT json->>'name' 
+        FROM database_entity de 
+        WHERE id = (SELECT er.toId 
+                    FROM entity_relationship er 
+                    WHERE er.fromId = de2.id 
+                    AND er.toEntity = 'database' 
+                    LIMIT 1)
+    )
+    )), 
+    '{connection,config,ingestAllDatabases}', 
+    'true'::jsonb
+)
+WHERE de2.serviceType = 'Mssql' 
+AND json->>'{connection,config,database}' IS NULL;
+
+-- remove keyfile from clickhouse
+UPDATE dbservice_entity
+SET json = json #-'{connection,config,keyfile}'
+WHERE serviceType = 'Clickhouse';
+
+-- Clean old test connections
+TRUNCATE automations_workflow;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v015__update_server_change_log.sql

@@ -0,0 +1,2 @@
+-- This column is already created in v000__create_server_change_log.sql
+-- ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics jsonb;

bootstrap/sql/migrations/native/1.1.0/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,43 @@
+ALTER TABLE bot_entity ADD UNIQUE (nameHash);
+ALTER TABLE chart_entity ADD UNIQUE (fqnHash);
+ALTER TABLE classification ADD UNIQUE (nameHash);
+ALTER TABLE storage_container_entity ADD UNIQUE (fqnHash);
+ALTER TABLE dashboard_data_model_entity ADD UNIQUE (fqnHash);
+ALTER TABLE dashboard_entity ADD UNIQUE (fqnHash);
+ALTER TABLE dashboard_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE data_insight_chart ADD UNIQUE (fqnHash);
+ALTER TABLE database_entity ADD UNIQUE (fqnHash);
+ALTER TABLE database_schema_entity ADD UNIQUE (fqnHash);
+ALTER TABLE dbservice_entity ADD UNIQUE (nameHash);
+ALTER TABLE event_subscription_entity ADD UNIQUE (nameHash);
+ALTER TABLE glossary_entity ADD UNIQUE (nameHash);
+ALTER TABLE glossary_term_entity ADD UNIQUE (fqnHash);
+ALTER TABLE ingestion_pipeline_entity ADD UNIQUE (fqnHash);
+ALTER TABLE kpi_entity ADD UNIQUE (nameHash);
+ALTER TABLE messaging_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE metadata_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE metric_entity ADD UNIQUE (fqnHash);
+ALTER TABLE ml_model_entity ADD UNIQUE (fqnHash);
+ALTER TABLE mlmodel_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE pipeline_entity ADD UNIQUE (fqnHash);
+ALTER TABLE pipeline_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE policy_entity ADD UNIQUE (fqnHash);
+ALTER TABLE query_entity ADD UNIQUE (nameHash);
+ALTER TABLE report_entity ADD UNIQUE (fqnHash);
+ALTER TABLE role_entity ADD UNIQUE (nameHash);
+ALTER TABLE storage_service_entity ADD UNIQUE (nameHash);
+ALTER TABLE table_entity ADD UNIQUE (fqnHash);
+ALTER TABLE tag ADD UNIQUE (fqnHash);
+ALTER TABLE team_entity ADD UNIQUE (nameHash);
+ALTER TABLE test_case ADD UNIQUE (fqnHash);
+ALTER TABLE test_connection_definition ADD UNIQUE (nameHash);
+ALTER TABLE test_definition ADD UNIQUE (nameHash);
+ALTER TABLE test_suite ADD UNIQUE (nameHash);
+ALTER TABLE topic_entity ADD UNIQUE (fqnHash);
+ALTER TABLE type_entity ADD UNIQUE (nameHash);
+ALTER TABLE user_entity ADD UNIQUE (nameHash);
+ALTER TABLE web_analytic_event ADD UNIQUE (fqnHash);
+ALTER TABLE automations_workflow ADD UNIQUE (nameHash);
+ALTER TABLE entity_extension_time_series DROP COLUMN entityFQN;
+ALTER TABLE field_relationship DROP KEY `PRIMARY`,ADD CONSTRAINT  `field_relationship_primary` PRIMARY KEY(fromFQNHash, toFQNHash, relation), MODIFY fromFQN VARCHAR(2096) NOT NULL, MODIFY toFQN VARCHAR(2096) NOT NULL;
+ALTER TABLE tag_usage DROP index `source`, DROP COLUMN targetFQN, ADD UNIQUE KEY `tag_usage_key` (source, tagFQNHash, targetFQNHash);

bootstrap/sql/migrations/native/1.1.0/mysql/schemaChanges.sql

@@ -0,0 +1,44 @@
+UPDATE query_entity SET json = JSON_INSERT(json, '$.fullyQualifiedName', JSON_EXTRACT(json, '$.name')) WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') IS NULL;
+ALTER TABLE bot_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE chart_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE classification DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE storage_container_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE dashboard_data_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE dashboard_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE dashboard_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE data_insight_chart DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
+ALTER TABLE database_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE database_schema_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE dbservice_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE event_subscription_entity DROP KEY `name`,  ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE glossary_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE glossary_term_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE ingestion_pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE kpi_entity  DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE messaging_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE metadata_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE metric_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE ml_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE mlmodel_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE pipeline_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE policy_entity DROP COLUMN fullyQualifiedName,  ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE query_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE report_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE role_entity DROP KEY `name`,  ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE storage_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE table_entity DROP COLUMN fullyQualifiedName,  ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE tag DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE team_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE test_case DROP COLUMN fullyQualifiedName,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
+ALTER TABLE test_connection_definition ADD COLUMN nameHash VARCHAR(256) NOT NULL,ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE test_definition DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE test_suite DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE topic_entity DROP COLUMN fullyQualifiedName,  ADD COLUMN fqnHash VARCHAR(768) NOT NULL,  ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
+ALTER TABLE type_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE user_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE web_analytic_event DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
+ALTER TABLE automations_workflow DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
+ALTER TABLE field_relationship ADD COLUMN fromFQNHash VARCHAR(382), ADD COLUMN toFQNHash VARCHAR(382), DROP INDEX from_index, DROP INDEX to_index, ADD INDEX from_fqnhash_index(fromFQNHash, relation), ADD INDEX to_fqnhash_index(toFQNHash, relation);
+ALTER TABLE entity_extension_time_series ADD COLUMN entityFQNHash VARCHAR (768) NOT NULL;
+ALTER TABLE tag_usage ADD COLUMN tagFQNHash VARCHAR(382), ADD COLUMN targetFQNHash VARCHAR(382);

bootstrap/sql/migrations/native/1.1.0/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,43 @@
+ALTER TABLE bot_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE chart_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE classification ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE storage_container_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE dashboard_data_model_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE dashboard_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE dashboard_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE data_insight_chart ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE database_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE database_schema_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE dbservice_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE event_subscription_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE glossary_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE glossary_term_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE ingestion_pipeline_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE kpi_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE messaging_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE metadata_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE metric_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE ml_model_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE mlmodel_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE pipeline_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE pipeline_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE policy_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE query_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE report_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE role_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE storage_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE table_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE tag ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE team_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE test_case ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE test_connection_definition ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE test_definition ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE test_suite ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE topic_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE type_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE user_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE web_analytic_event ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
+ALTER TABLE automations_workflow ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
+ALTER TABLE field_relationship DROP CONSTRAINT field_relationship_pkey, ADD CONSTRAINT field_relationship_pkey PRIMARY KEY(fromFQNHash, toFQNHash, relation), ALTER fromFQN TYPE VARCHAR(2096), ALTER toFQN TYPE VARCHAR(2096);
+ALTER TABLE entity_extension_time_series DROP COLUMN entityFQN, ALTER COLUMN entityFQNHash SET NOT NULL;
+ALTER TABLE tag_usage DROP CONSTRAINT tag_usage_source_tagfqn_targetfqn_key, DROP COLUMN targetFQN, ADD UNIQUE (source, tagFQNHash, targetFQNHash);

bootstrap/sql/migrations/native/1.1.0/postgres/schemaChanges.sql

@@ -0,0 +1,47 @@
+UPDATE query_entity SET json = jsonb_set(json::jsonb, '{fullyQualifiedName}', json#>'{name}') WHERE json#>'{fullyQualifiedName}' IS NULL;
+ALTER TABLE bot_entity DROP CONSTRAINT bot_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE chart_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE classification  DROP CONSTRAINT tag_category_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE storage_container_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE dashboard_data_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE dashboard_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE dashboard_service_entity DROP CONSTRAINT dashboard_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE data_insight_chart DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768);
+ALTER TABLE database_entity  DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE database_schema_entity  DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE dbservice_entity DROP CONSTRAINT dbservice_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE event_subscription_entity DROP CONSTRAINT event_subscription_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE glossary_entity DROP CONSTRAINT glossary_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE glossary_term_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE ingestion_pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE kpi_entity  DROP CONSTRAINT kpi_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE messaging_service_entity DROP CONSTRAINT messaging_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE metadata_service_entity DROP CONSTRAINT metadata_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE metric_entity  DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE ml_model_entity  DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE mlmodel_service_entity DROP CONSTRAINT mlmodel_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE pipeline_entity  DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE pipeline_service_entity DROP CONSTRAINT pipeline_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE policy_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE query_entity DROP CONSTRAINT query_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE report_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE role_entity DROP CONSTRAINT role_entity_name_key,  ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE storage_service_entity DROP CONSTRAINT storage_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE table_entity DROP COLUMN fullyQualifiedName,  ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE tag DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE team_entity DROP CONSTRAINT team_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE test_case DROP COLUMN fullyQualifiedName, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, ADD COLUMN fqnHash VARCHAR(768);
+ALTER TABLE test_connection_definition ADD COLUMN nameHash VARCHAR(256), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE test_definition DROP CONSTRAINT test_definition_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE test_suite DROP CONSTRAINT test_suite_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE topic_entity  DROP COLUMN fullyQualifiedName,  ADD COLUMN fqnHash VARCHAR(256), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
+ALTER TABLE type_entity DROP CONSTRAINT type_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE user_entity DROP CONSTRAINT user_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
+ALTER TABLE web_analytic_event DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768);
+ALTER TABLE automations_workflow DROP CONSTRAINT automations_workflow_name_key,  ADD COLUMN nameHash VARCHAR(256);
+DROP INDEX field_relationship_from_index, field_relationship_to_index;
+ALTER TABLE field_relationship ADD COLUMN fromFQNHash VARCHAR(382), ADD COLUMN toFQNHash VARCHAR(382);
+CREATE INDEX IF NOT EXISTS field_relationship_from_index ON field_relationship(fromFQNHash, relation);
+CREATE INDEX IF NOT EXISTS field_relationship_to_index ON field_relationship(toFQNHash, relation);
+ALTER TABLE entity_extension_time_series ADD COLUMN entityFQNHash VARCHAR (768);
+ALTER TABLE tag_usage ADD COLUMN tagFQNHash VARCHAR(382), ADD COLUMN targetFQNHash VARCHAR(382);

bootstrap/sql/migrations/native/1.1.1/mysql/schemaChanges.sql

@@ -0,0 +1 @@
+ALTER TABLE test_suite CHANGE COLUMN nameHash fqnHash VARCHAR(256);

bootstrap/sql/migrations/native/1.1.1/postgres/schemaChanges.sql

@@ -0,0 +1 @@
+ALTER TABLE test_suite RENAME COLUMN nameHash TO fqnHash;

bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql

@@ -0,0 +1,23 @@
+-- queries to rename params to connectionOptions for trino
+UPDATE dbservice_entity
+SET json = JSON_SET(
+        JSON_REMOVE(json, '$.connection.config.connectionOptions'),
+        '$.connection.config.connectionOptions',
+        JSON_EXTRACT(json, '$.connection.config.params')
+    )
+WHERE serviceType = 'Trino';
+
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.params')
+WHERE serviceType = 'Trino';
+
+-- Modify migrations for service connection of trino to move password under authType
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+        JSON_REMOVE(json, '$.connection.config.password'),
+        '$.connection.config.authType',
+        JSON_OBJECT(),
+        '$.connection.config.authType.password',
+        JSON_EXTRACT(json, '$.connection.config.password'))
+where serviceType = 'Trino'
+  AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;

bootstrap/sql/migrations/native/1.1.2/postgres/schemaChanges.sql

@@ -0,0 +1,23 @@
+-- queries to rename params to connectionOptions for trino
+UPDATE dbservice_entity
+SET json = jsonb_set(
+    json,
+    '{connection,config,connectionOptions}',
+    jsonb_extract_path(json, 'connection', 'config', 'params'),
+    true
+)
+WHERE serviceType = 'Trino';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,params}'
+where json #> '{serviceType}' in ('"Trino"');
+
+-- Modify migrations for service connection of trino to move password under authType
+UPDATE dbservice_entity
+SET json =  jsonb_set(
+json #-'{connection,config,password}',
+'{connection,config,authType}',
+jsonb_build_object('password',json#>'{connection,config,password}')
+)
+WHERE serviceType = 'Trino'
+  and json#>'{connection,config,password}' is not null;

bootstrap/sql/migrations/native/1.1.5/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,50 @@
+START TRANSACTION;
+-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> '$.id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+		FROM (
+			SELECT
+			*
+			FROM report_data_time_series rdts
+			WHERE json ->> '$.reportDataType' = 'WebAnalyticEntityViewReportData'
+		) duplicates
+		ORDER BY `date` DESC, `timestamp` DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> '$.id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+		FROM (
+			SELECT
+			*
+			FROM report_data_time_series rdts
+			WHERE json ->> '$.reportDataType' = 'EntityReportData'
+		) duplicates
+		ORDER BY `date` DESC, `timestamp` DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> '$.id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+		FROM (
+			SELECT
+			*
+			FROM report_data_time_series rdts
+			WHERE json ->> '$.reportDataType' = 'WebAnalyticUserActivityReportData'
+		) duplicates
+		ORDER BY `date` DESC, `timestamp` DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+COMMIT;

bootstrap/sql/migrations/native/1.1.5/mysql/schemaChanges.sql

@@ -0,0 +1,120 @@
+-- Update table and column profile timestamps to be in milliseconds
+UPDATE entity_extension_time_series
+    SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.timestamp'),
+    '$.timestamp',
+    JSON_EXTRACT(json, '$.timestamp') * 1000
+    )
+WHERE
+    extension  in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
+;
+
+START TRANSACTION;
+-- Create report data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS report_data_time_series (
+    entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSON NOT NULL,
+    timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+    date DATE GENERATED ALWAYS AS (FROM_UNIXTIME((json ->> '$.timestamp') DIV 1000)) NOT NULL,
+    INDEX report_data_time_series_point_ts (timestamp),
+    INDEX report_data_time_series_date (date)
+);
+
+INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'reportData.reportDataResult';
+COMMIT;
+
+START TRANSACTION;
+-- Create profiler data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS profiler_data_time_series (
+    entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSON NOT NULL,
+    operation VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.operation') NULL,
+    timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+    UNIQUE profiler_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, operation, timestamp),
+    INDEX profiler_data_time_series_combined_id_ts (extension, timestamp)
+);
+
+INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+
+DELETE FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+COMMIT;
+
+START TRANSACTION;
+-- Create data quality data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
+    entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSON NOT NULL,
+    timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+    UNIQUE data_quality_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, timestamp),
+    INDEX data_quality_data_time_series_combined_id_ts (extension, timestamp)
+);
+
+INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+COMMIT;
+
+ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin,
+    ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
+ALTER TABLE field_relationship MODIFY COLUMN fromFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN toFQNHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE thread_entity MODIFY COLUMN entityLink VARCHAR(3072) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL;
+ALTER TABLE event_subscription_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE ingestion_pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE bot_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE user_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE team_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE role_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE policy_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE classification MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE tag MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE tag_usage MODIFY COLUMN tagFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN targetFQNHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE glossary_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE glossary_term_entity MODIFY fqnHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE type_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE web_analytic_event MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE data_insight_chart MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE kpi_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE test_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE test_connection_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE test_suite MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE test_case MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE dbservice_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE messaging_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE metadata_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE pipeline_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE dashboard_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE mlmodel_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE storage_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE database_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE database_schema_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE table_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE dashboard_data_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE dashboard_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE chart_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE ml_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE metric_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE query_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
+ALTER TABLE report_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE storage_container_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
+ALTER TABLE topic_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;

bootstrap/sql/migrations/native/1.1.5/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,53 @@
+BEGIN;
+-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> 'id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+		FROM (
+			SELECT
+			*,
+            DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+			FROM report_data_time_series rdts
+			WHERE json ->> 'reportDataType' = 'WebAnalyticEntityViewReportData'
+		) duplicates
+		ORDER BY date DESC, timestamp DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> 'id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+		FROM (
+			SELECT
+			*,
+            DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+			FROM report_data_time_series rdts
+			WHERE json ->> 'reportDataType' = 'EntityReportData'
+		) duplicates
+		ORDER BY date DESC, timestamp DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+	select ids FROM (
+		SELECT 
+		(json ->> 'id') AS ids,
+		DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+		FROM (
+			SELECT
+			*,
+            DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+			FROM report_data_time_series rdts
+			WHERE json ->> 'reportDataType' = 'WebAnalyticUserActivityReportData'
+		) duplicates
+		ORDER BY date DESC, timestamp DESC
+		) as dense_ranked
+		WHERE denseRank != 1
+);
+COMMIT;

bootstrap/sql/migrations/native/1.1.5/postgres/schemaChanges.sql

@@ -0,0 +1,83 @@
+-- Update table and column profile timestamps to be in milliseconds
+UPDATE entity_extension_time_series
+SET json = jsonb_set(
+	json,
+	'{timestamp}',
+	to_jsonb(cast(json#>'{timestamp}' as int8) *1000)
+)
+WHERE
+	extension  in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
+;
+
+BEGIN;
+-- Run the following SQL to update the schema in a transaction
+-- Create report data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS report_data_time_series (
+    entityFQNHash VARCHAR(768),
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSONB NOT NULL,
+    timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
+);
+CREATE INDEX IF NOT EXISTS report_data_time_series_point_ts ON report_data_time_series (timestamp);
+
+INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
+
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'reportData.reportDataResult';
+COMMIT;
+
+BEGIN;
+-- Create profiler data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS profiler_data_time_series (
+    entityFQNHash VARCHAR(768),
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSON NOT NULL,
+    operation VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'operation')::text) STORED NULL,
+    timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
+    CONSTRAINT profiler_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, operation, timestamp)
+);
+
+CREATE INDEX IF NOT EXISTS profiler_data_time_series_combined_id_ts ON profiler_data_time_series (extension, timestamp);
+
+INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+
+DELETE FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+COMMIT;
+
+BEGIN;
+-- Create profiler data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
+    entityFQNHash VARCHAR(768),
+    extension VARCHAR(256) NOT NULL,
+    jsonSchema VARCHAR(256) NOT NULL,
+    json JSON NOT NULL,
+    timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
+    CONSTRAINT data_quality_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, timestamp)
+);
+
+CREATE INDEX IF NOT EXISTS data_quality_data_time_series_combined_id_ts ON data_quality_data_time_series (extension, timestamp);
+
+INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+COMMIT;
+
+ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQNHash TYPE VARCHAR(768), ALTER COLUMN jsonSchema TYPE VARCHAR(50) , ALTER COLUMN extension TYPE VARCHAR(100) ,
+    ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
+ALTER TABLE field_relationship ALTER COLUMN fromFQNHash TYPE VARCHAR(768), ALTER COLUMN toFQNHash TYPE VARCHAR(768);
+ALTER TABLE thread_entity ALTER COLUMN entityLink TYPE VARCHAR(3072);
+ALTER TABLE tag_usage ALTER COLUMN tagFQNHash TYPE VARCHAR(768), ALTER COLUMN targetFQNHash TYPE VARCHAR(768);
+ALTER TABLE test_suite ALTER COLUMN fqnHash TYPE VARCHAR(768);

bootstrap/sql/migrations/native/1.10.0/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,63 @@
+-- Migration script to restructure Databricks connection configuration
+-- Move 'token' field from connection.config.token to connection.config.authType.token
+UPDATE dbservice_entity
+SET
+    json = JSON_SET (
+        JSON_REMOVE (json, '$.connection.config.token'),
+        '$.connection.config.authType',
+        JSON_OBJECT (
+            'token',
+            JSON_EXTRACT (json, '$.connection.config.token')
+        )
+    )
+WHERE
+    serviceType in ('Databricks', 'UnityCatalog')
+    AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.token');
+
+-- Migration to remove .png extension from appScreenshots in apps tables
+-- Part of fixing appScreenshots extension handling change
+
+-- Update apps_marketplace table - remove .png extension from appScreenshots only
+UPDATE apps_marketplace 
+SET json = JSON_SET(
+    json,
+    '$.appScreenshots',
+    JSON_EXTRACT(
+        REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
+        '$'
+    )
+)
+WHERE json IS NOT NULL
+  AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
+  AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
+  AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';
+
+-- Update installed_apps table - remove .png extension from appScreenshots only
+UPDATE installed_apps 
+SET json = JSON_SET(
+    json,
+    '$.appScreenshots',
+    JSON_EXTRACT(
+        REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
+        '$'
+    )
+)
+WHERE json IS NOT NULL
+  AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
+  AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
+  AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';
+
+-- Update apps_data_store table - remove .png extension from appScreenshots only
+UPDATE apps_data_store 
+SET json = JSON_SET(
+    json,
+    '$.appScreenshots',
+    JSON_EXTRACT(
+        REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
+        '$'
+    )
+)
+WHERE json IS NOT NULL
+  AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
+  AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
+  AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';

bootstrap/sql/migrations/native/1.10.0/mysql/schemaChanges.sql

@@ -0,0 +1,83 @@
+-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration
+UPDATE openmetadata_settings
+SET json = JSON_ARRAY_APPEND(
+    json,
+    '$.entitySemantics',
+    JSON_OBJECT(
+        'name', 'Data Product Domain Validation',
+        'description', 'Validates that Data Products assigned to an entity match the entity''s domains.',
+        'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}',
+        'enabled', true,
+        'provider', 'system'
+    )
+)
+WHERE configType = 'entityRulesSettings'
+  AND JSON_EXTRACT(json, '$.entitySemantics') IS NOT NULL
+  AND NOT JSON_CONTAINS(
+    JSON_EXTRACT(json, '$.entitySemantics[*].name'),
+    JSON_QUOTE('Data Product Domain Validation')
+  );
+
+-- Add virtual column for customUnitOfMeasurement
+ALTER TABLE metric_entity
+ADD COLUMN customUnitOfMeasurement VARCHAR(256)
+GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.customUnitOfMeasurement'))) VIRTUAL;
+-- Add index on the virtual column
+CREATE INDEX idx_metric_custom_unit ON metric_entity(customUnitOfMeasurement);
+
+-- Fetch updated searchSettings
+DELETE FROM openmetadata_settings WHERE configType = 'searchSettings';
+
+-- Create notification_template_entity table following OpenMetadata patterns
+CREATE TABLE IF NOT EXISTS notification_template_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.id'))) STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.name'))) VIRTUAL NOT NULL,
+    fqnHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.updatedAt'))) VIRTUAL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.updatedBy'))) VIRTUAL NOT NULL,
+    deleted TINYINT(1) GENERATED ALWAYS AS (json_extract(json, '$.deleted')) VIRTUAL,
+    provider VARCHAR(32) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.provider'))) VIRTUAL,
+
+    PRIMARY KEY (id),
+    UNIQUE KEY fqnHash (fqnHash),
+    INDEX idx_notification_template_name (name),
+    INDEX idx_notification_template_provider (provider)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+-- Optimize table listing queries by indexing the schema hash prefix
+ALTER TABLE table_entity
+ADD COLUMN databaseSchemaHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin
+GENERATED ALWAYS AS (SUBSTRING_INDEX(fqnHash, '.', 3)) STORED;
+
+CREATE INDEX idx_table_entity_schema_listing
+ON table_entity (deleted, databaseSchemaHash, name, id);
+
+-- Optimize stored procedure listing queries by indexing the schema hash prefix
+ALTER TABLE stored_procedure_entity
+ADD COLUMN databaseSchemaHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin
+GENERATED ALWAYS AS (SUBSTRING_INDEX(fqnHash, '.', 3)) STORED;
+
+CREATE INDEX idx_stored_procedure_schema_listing
+ON stored_procedure_entity (deleted, databaseSchemaHash, name, id);
+
+-- Recognizer Feedback Storage
+-- Store user feedback on auto-applied tags to improve recognition accuracy
+CREATE TABLE IF NOT EXISTS recognizer_feedback_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.id'))) STORED NOT NULL,
+    entityLink VARCHAR(512) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.entityLink'))) VIRTUAL NOT NULL,
+    tagFQN VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.tagFQN'))) VIRTUAL NOT NULL,
+    feedbackType VARCHAR(50) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.feedbackType'))) VIRTUAL NOT NULL,
+    status VARCHAR(20) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.status'))) VIRTUAL,
+    createdBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.createdBy'))) VIRTUAL NOT NULL,
+    createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.createdAt'))) VIRTUAL NOT NULL,
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    INDEX idx_feedback_entity (entityLink),
+    INDEX idx_feedback_tag (tagFQN),
+    INDEX idx_feedback_status (status),
+    INDEX idx_feedback_created (createdAt)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+ALTER TABLE tag_usage
+ADD COLUMN reason TEXT;

bootstrap/sql/migrations/native/1.10.0/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,51 @@
+-- Migration script to restructure Databricks connection configuration
+-- Move 'token' field from connection.config.token to connection.config.authType.token
+
+UPDATE dbservice_entity
+SET json = jsonb_set(
+    json #- '{connection,config,token}',
+    '{connection,config,authType}',
+    jsonb_build_object('token', json #> '{connection,config,token}'),
+    true
+)
+WHERE serviceType in ('Databricks', 'UnityCatalog')
+    AND jsonb_exists(json -> 'connection' -> 'config', 'token');
+
+-- Migration to remove .png extension from appScreenshots in apps tables
+-- Part of fixing appScreenshots extension handling change
+
+-- Update apps_marketplace table - remove .png extension from appScreenshots only
+UPDATE apps_marketplace 
+SET json = jsonb_set(
+    json,
+    '{appScreenshots}',
+    REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
+)
+WHERE json IS NOT NULL
+  AND json -> 'appScreenshots' IS NOT NULL
+  AND jsonb_array_length(json -> 'appScreenshots') > 0
+  AND (json -> 'appScreenshots')::text LIKE '%.png%';
+
+-- Update installed_apps table - remove .png extension from appScreenshots only
+UPDATE installed_apps 
+SET json = jsonb_set(
+    json,
+    '{appScreenshots}',
+    REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
+)
+WHERE json IS NOT NULL
+  AND json -> 'appScreenshots' IS NOT NULL
+  AND jsonb_array_length(json -> 'appScreenshots') > 0
+  AND (json -> 'appScreenshots')::text LIKE '%.png%';
+
+-- Update apps_data_store table - remove .png extension from appScreenshots only
+UPDATE apps_data_store 
+SET json = jsonb_set(
+    json::jsonb,
+    '{appScreenshots}',
+    REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
+)
+WHERE json IS NOT NULL
+  AND json -> 'appScreenshots' IS NOT NULL
+  AND json_array_length((json -> 'appScreenshots')::json) > 0
+  AND (json -> 'appScreenshots')::text LIKE '%.png%';

bootstrap/sql/migrations/native/1.10.0/postgres/schemaChanges.sql

@@ -0,0 +1,104 @@
+-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration
+UPDATE openmetadata_settings
+SET json = jsonb_set(
+    json,
+    '{entitySemantics}',
+    (json->'entitySemantics') || jsonb_build_object(
+        'name', 'Data Product Domain Validation',
+        'description', 'Validates that Data Products assigned to an entity match the entity''s domains.',
+        'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}',
+        'enabled', true,
+        'provider', 'system'
+    )::jsonb,
+    true
+)
+WHERE configtype = 'entityRulesSettings'
+  AND json->'entitySemantics' IS NOT NULL
+  AND NOT EXISTS (
+    SELECT 1
+    FROM jsonb_array_elements(json->'entitySemantics') AS rule
+    WHERE rule->>'name' = 'Data Product Domain Validation'
+  );
+
+-- Add generated column for customUnitOfMeasurement
+ALTER TABLE metric_entity
+ADD COLUMN customUnitOfMeasurement VARCHAR(256)
+GENERATED ALWAYS AS ((json->>'customUnitOfMeasurement')::VARCHAR(256)) STORED;
+-- Add index on the column
+CREATE INDEX idx_metric_custom_unit ON metric_entity(customUnitOfMeasurement);
+
+-- Fetch updated searchSettings
+DELETE FROM openmetadata_settings WHERE configType = 'searchSettings';
+
+-- Create notification_template_entity table following OpenMetadata patterns
+CREATE TABLE IF NOT EXISTS notification_template_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    fqnHash VARCHAR(768) NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    provider VARCHAR(32) GENERATED ALWAYS AS (json ->> 'provider') STORED,
+
+    PRIMARY KEY (id),
+    UNIQUE (fqnHash)
+);
+
+CREATE INDEX IF NOT EXISTS idx_notification_template_name ON notification_template_entity(name);
+CREATE INDEX IF NOT EXISTS idx_notification_template_provider ON notification_template_entity(provider);
+
+-- Optimize table listing queries by indexing the schema hash prefix
+ALTER TABLE table_entity
+ADD COLUMN IF NOT EXISTS databaseSchemaHash VARCHAR(768)
+GENERATED ALWAYS AS (
+  rtrim(
+    split_part(fqnhash, '.', 1) || '.' ||
+    split_part(fqnhash, '.', 2) || '.' ||
+    split_part(fqnhash, '.', 3),
+    '.'
+  )
+) STORED;
+
+CREATE INDEX IF NOT EXISTS idx_table_entity_schema_listing
+ON table_entity (deleted, databaseSchemaHash, name, id);
+
+-- Optimize stored procedure listing queries by indexing the schema hash prefix
+ALTER TABLE stored_procedure_entity
+ADD COLUMN IF NOT EXISTS databaseSchemaHash VARCHAR(768)
+GENERATED ALWAYS AS (
+  rtrim(
+    split_part(fqnhash, '.', 1) || '.' ||
+    split_part(fqnhash, '.', 2) || '.' ||
+    split_part(fqnhash, '.', 3),
+    '.'
+  )
+) STORED;
+
+DROP INDEX IF EXISTS idx_stored_procedure_entity_deleted_name_id;
+
+CREATE INDEX IF NOT EXISTS idx_stored_procedure_schema_listing
+ON stored_procedure_entity (deleted, databaseSchemaHash, name, id);
+
+
+-- Recognizer Feedback Storage
+-- Store user feedback on auto-applied tags to improve recognition accuracy
+CREATE TABLE IF NOT EXISTS recognizer_feedback_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL,
+    tagFQN VARCHAR(256) GENERATED ALWAYS AS (json ->> 'tagFQN') STORED NOT NULL,
+    feedbackType VARCHAR(50) GENERATED ALWAYS AS (json ->> 'feedbackType') STORED NOT NULL,
+    status VARCHAR(20) GENERATED ALWAYS AS (json ->> 'status') STORED,
+    createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
+    createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'createdAt')::bigint) STORED NOT NULL,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_feedback_entity ON recognizer_feedback_entity(entityLink);
+CREATE INDEX IF NOT EXISTS idx_feedback_tag ON recognizer_feedback_entity(tagFQN);
+CREATE INDEX IF NOT EXISTS idx_feedback_status ON recognizer_feedback_entity(status);
+CREATE INDEX IF NOT EXISTS idx_feedback_created ON recognizer_feedback_entity(createdAt);
+
+ALTER TABLE tag_usage
+ADD COLUMN reason TEXT;

bootstrap/sql/migrations/native/1.10.2/mysql/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- DO NOT RENAME OR DELETE THIS FILE
+-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1102

bootstrap/sql/migrations/native/1.10.2/postgres/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- DO NOT RENAME OR DELETE THIS FILE
+-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1102

bootstrap/sql/migrations/native/1.10.3/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,8 @@
+UPDATE test_definition
+SET json = JSON_ARRAY_INSERT(
+    json,
+    CONCAT('$.supportedDataTypes[', JSON_LENGTH(json, '$.supportedDataTypes'), ']'),
+    'NUMERIC'
+)
+WHERE JSON_CONTAINS(json->'$.supportedDataTypes', '"NUMBER"')
+  AND NOT JSON_CONTAINS(json->'$.supportedDataTypes', '"NUMERIC"');

bootstrap/sql/migrations/native/1.10.3/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,9 @@
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '"NUMERIC"'::jsonb,
+    true
+)
+WHERE json->'supportedDataTypes' @> '"NUMBER"'::jsonb
+  AND NOT (json->'supportedDataTypes' @> '"NUMERIC"'::jsonb);

bootstrap/sql/migrations/native/1.10.4/mysql/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- Delete searchSettings to force reload from packaged searchSettings.json with field-based aggregations
+DELETE FROM openmetadata_settings WHERE configType='searchSettings';

bootstrap/sql/migrations/native/1.10.4/postgres/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- Delete searchSettings to force reload from packaged searchSettings.json with field-based aggregations
+DELETE FROM openmetadata_settings WHERE configType='searchSettings';

bootstrap/sql/migrations/native/1.10.5/mysql/schemaChanges.sql

@@ -0,0 +1,22 @@
+-- Increase Flowable ACTIVITY_ID_ column size to support longer user-defined workflow node names
+ALTER TABLE ACT_RU_EVENT_SUBSCR MODIFY ACTIVITY_ID_ varchar(255);
+
+-- Update workflow settings with new job acquisition interval settings
+UPDATE openmetadata_settings
+SET json = JSON_SET(
+    json,
+    '$.executorConfiguration.asyncJobAcquisitionInterval', 60000,
+    '$.executorConfiguration.timerJobAcquisitionInterval', 60000
+)
+WHERE configType = 'workflowSettings'
+  AND JSON_EXTRACT(json, '$.executorConfiguration') IS NOT NULL;
+
+-- TRUNCATE FLOWABLE HISTORY TABLES
+TRUNCATE TABLE ACT_HI_PROCINST;
+TRUNCATE TABLE ACT_HI_ACTINST;
+TRUNCATE TABLE ACT_HI_TASKINST;
+TRUNCATE TABLE ACT_HI_VARINST;
+TRUNCATE TABLE ACT_HI_DETAIL;
+TRUNCATE TABLE ACT_HI_COMMENT;
+TRUNCATE TABLE ACT_HI_ATTACHMENT;
+TRUNCATE TABLE ACT_HI_IDENTITYLINK;

bootstrap/sql/migrations/native/1.10.5/postgres/schemaChanges.sql

@@ -0,0 +1,28 @@
+-- Increase Flowable ACTIVITY_ID_ column size to support longer user-defined workflow node names
+ALTER TABLE ACT_RU_EVENT_SUBSCR ALTER COLUMN ACTIVITY_ID_ TYPE varchar(255);
+
+-- Update workflow settings with new job acquisition interval settings
+UPDATE openmetadata_settings
+SET json = jsonb_set(
+    jsonb_set(
+        json,
+        '{executorConfiguration,asyncJobAcquisitionInterval}',
+        '60000',
+        true
+    ),
+    '{executorConfiguration,timerJobAcquisitionInterval}',
+    '60000',
+    true
+)
+WHERE configtype = 'workflowSettings'
+  AND json->'executorConfiguration' IS NOT NULL;
+
+-- TRUNCATE FLOWABLE HISTORY TABLES
+TRUNCATE TABLE ACT_HI_PROCINST;
+TRUNCATE TABLE ACT_HI_ACTINST;
+TRUNCATE TABLE ACT_HI_TASKINST;
+TRUNCATE TABLE ACT_HI_VARINST;
+TRUNCATE TABLE ACT_HI_DETAIL;
+TRUNCATE TABLE ACT_HI_COMMENT;
+TRUNCATE TABLE ACT_HI_ATTACHMENT;
+TRUNCATE TABLE ACT_HI_IDENTITYLINK;

bootstrap/sql/migrations/native/1.10.6/mysql/schemaChanges.sql

@@ -0,0 +1,23 @@
+-- Migrate Airbyte connections from flat username/password to auth object structure
+UPDATE pipeline_service_entity
+SET
+    json = JSON_SET (
+        JSON_REMOVE (
+            json,
+            '$.connection.config.username',
+            '$.connection.config.password'
+        ),
+        '$.connection.config.auth',
+        JSON_OBJECT (
+            'username',
+            JSON_UNQUOTE (
+                JSON_EXTRACT (json, '$.connection.config.username')
+            ),
+            'password',
+            JSON_UNQUOTE (
+                JSON_EXTRACT (json, '$.connection.config.password')
+            )
+        )
+    )
+WHERE
+    JSON_EXTRACT (json, '$.serviceType') = 'Airbyte';

bootstrap/sql/migrations/native/1.10.6/postgres/schemaChanges.sql

@@ -0,0 +1,12 @@
+-- Migrate Airbyte connections from flat username/password to auth object structure
+UPDATE pipeline_service_entity
+SET json = jsonb_set(
+    json #- '{connection,config,username}' #- '{connection,config,password}',
+    '{connection,config,auth}',
+    jsonb_build_object(
+        'username', json#>>'{connection,config,username}',
+        'password', json#>>'{connection,config,password}'
+    ),
+    true
+)
+WHERE json->>'serviceType' = 'Airbyte';

bootstrap/sql/migrations/native/1.10.7/mysql/schemaChanges.sql

@@ -0,0 +1,9 @@
+-- TRUNCATE FLOWABLE HISTORY TABLES
+TRUNCATE TABLE ACT_HI_PROCINST;
+TRUNCATE TABLE ACT_HI_ACTINST;
+TRUNCATE TABLE ACT_HI_TASKINST;
+TRUNCATE TABLE ACT_HI_VARINST;
+TRUNCATE TABLE ACT_HI_DETAIL;
+TRUNCATE TABLE ACT_HI_COMMENT;
+TRUNCATE TABLE ACT_HI_ATTACHMENT;
+TRUNCATE TABLE ACT_HI_IDENTITYLINK;

bootstrap/sql/migrations/native/1.10.7/postgres/schemaChanges.sql

@@ -0,0 +1,9 @@
+-- TRUNCATE FLOWABLE HISTORY TABLES
+TRUNCATE TABLE ACT_HI_PROCINST;
+TRUNCATE TABLE ACT_HI_ACTINST;
+TRUNCATE TABLE ACT_HI_TASKINST;
+TRUNCATE TABLE ACT_HI_VARINST;
+TRUNCATE TABLE ACT_HI_DETAIL;
+TRUNCATE TABLE ACT_HI_COMMENT;
+TRUNCATE TABLE ACT_HI_ATTACHMENT;
+TRUNCATE TABLE ACT_HI_IDENTITYLINK;

bootstrap/sql/migrations/native/1.10.8/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,6 @@
+-- Fix emailConfiguration templates field for users affected by v1.8.0 bug
+-- See commit: https://github.com/open-metadata/OpenMetadata/commit/a50cddf665544b0934ce79a539aecbb00bd541c8
+-- This migration ensures all emailConfiguration records have the correct 'openmetadata' value
+UPDATE openmetadata_settings
+SET json = JSON_SET(json, '$.templates', 'openmetadata')
+WHERE configType = 'emailConfiguration';

bootstrap/sql/migrations/native/1.10.8/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,6 @@
+-- Fix emailConfiguration templates field for users affected by v1.8.0 bug
+-- See commit: https://github.com/open-metadata/OpenMetadata/commit/a50cddf665544b0934ce79a539aecbb00bd541c8
+-- This migration ensures all emailConfiguration records have the correct 'openmetadata' value
+UPDATE openmetadata_settings
+SET json = jsonb_set(json, '{templates}', '"openmetadata"')
+WHERE configType = 'emailConfiguration';

bootstrap/sql/migrations/native/1.11.0/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,104 @@
+-- Correct the table diff test definition
+-- This is to include the new parameter table2.keyColumns
+UPDATE test_definition
+SET json = JSON_SET(
+        json,
+        '$.parameterDefinition',
+        JSON_ARRAY(
+                JSON_OBJECT(
+                        'name', 'keyColumns',
+                        'displayName', 'Table 1''s key columns',
+            'description', 'The columns to use as the key for the comparison. If not provided, it will be resolved from the primary key or unique columns. The tuples created from the key columns must be unique.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        JSON_OBJECT(
+            'name', 'table2',
+            'displayName', 'Table 2',
+            'description', 'Fully qualified name of the table to compare against.',
+            'dataType', 'STRING',
+            'required', true
+        ),
+        JSON_OBJECT(
+            'name', 'table2.keyColumns',
+            'displayName', 'Table 2''s key columns',
+                        'description', 'The columns in table 2 to use as comparison. If not provided, it will default to `Key Columns`, risking errors if the key columns'' names have changed.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        JSON_OBJECT(
+            'name', 'threshold',
+            'displayName', 'Threshold',
+            'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
+            'dataType', 'NUMBER',
+            'required', false
+        ),
+        JSON_OBJECT(
+            'name', 'useColumns',
+            'displayName', 'Use Columns',
+            'description', 'Limits the scope of the test to this list of columns. If not provided, all columns will be used except the key columns.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        JSON_OBJECT(
+            'name', 'where',
+            'displayName', 'SQL Where Clause',
+            'description', 'Use this where clause to filter the rows to compare.',
+            'dataType', 'STRING',
+            'required', false
+        ),
+        JSON_OBJECT(
+            'name', 'caseSensitiveColumns',
+            'displayName', 'Case sensitive columns',
+            'description', 'Use case sensitivity when comparing the columns.',
+            'dataType', 'BOOLEAN',
+            'required', false
+        )
+    ),
+    '$.version',
+    0.2
+)
+WHERE name = 'tableDiff';
+
+-- Update DataRetentionApplication add profileDataRetentionPeriod and testCaseResultsRetentionPeriod
+UPDATE installed_apps
+SET json = JSON_SET(
+    json,
+    '$.appConfiguration.testCaseResultsRetentionPeriod', 1440,
+    '$.appConfiguration.profileDataRetentionPeriod', 1440
+)
+WHERE JSON_EXTRACT(json, '$.name') = 'DataRetentionApplication';
+
+UPDATE notification_template_entity
+SET json = JSON_REMOVE(json, '$.defaultTemplateChecksum')
+WHERE JSON_CONTAINS_PATH(json, 'one', '$.defaultTemplateChecksum');
+
+-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAIQualityAgentApplication and CollateAITierAgentApplication
+UPDATE apps_marketplace
+SET json = JSON_SET(
+    JSON_SET(
+        json,
+        '$.appType',
+        'external'
+    ),
+    '$.sourcePythonClass',
+    'metadata.applications.dynamic_agent.app.DynamicAgentApp'
+)
+WHERE json->>'$.name' IN  ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
+    AND json->>'$.appType' = 'internal' AND json->>'$.sourcePythonClass' IS NULL;
+
+-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAITierAgentApplication and CollateAIQualityAgentApplication
+UPDATE installed_apps
+SET json = JSON_SET(
+    json,
+    '$.appType', 'external',
+    '$.sourcePythonClass', 'metadata.applications.dynamic_agent.app.DynamicAgentApp'
+)
+WHERE json->>'$.name' IN ('CollateAITierAgentApplication', 'CollateAIQualityAgentApplication')
+  AND json->>'$.appType' = 'internal' AND json->>'$.sourcePythonClass' IS NULL;
+  
+-- Remove bot form App entity    
+UPDATE installed_apps SET json = JSON_REMOVE(json, '$.bot') WHERE JSON_CONTAINS_PATH(json, 'one', '$.bot');
+
+-- Remove SearchIndexingApplication past runs
+delete from apps_extension_time_series where appname = 'SearchIndexingApplication';

bootstrap/sql/migrations/native/1.11.0/mysql/schemaChanges.sql

@@ -0,0 +1,140 @@
+-- Test Case Dimension Results Time Series Table
+CREATE TABLE IF NOT EXISTS test_case_dimension_results_time_series (
+  entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+  extension VARCHAR(256) NOT NULL DEFAULT 'testCase.dimensionResult',
+  jsonSchema VARCHAR(256) NOT NULL,
+  json JSON NOT NULL,
+  id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.id'))) STORED NOT NULL,
+  testCaseResultId VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseResultId'))) STORED NOT NULL,
+  dimensionKey VARCHAR(512) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.dimensionKey'))) STORED NOT NULL,
+  dimensionName VARCHAR(256) GENERATED ALWAYS AS (SUBSTRING_INDEX(json_unquote(json_extract(json,'$.dimensionKey')), '=', 1)) STORED,
+  timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.timestamp'))) STORED NOT NULL,
+  testCaseStatus VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseStatus'))) STORED,
+  UNIQUE KEY test_case_dimension_results_unique_constraint (entityFQNHash, dimensionKey, timestamp),
+  INDEX test_case_dimension_results_main (entityFQNHash, timestamp, dimensionKey),
+  INDEX test_case_dimension_results_dimension_name (entityFQNHash, dimensionName, timestamp),
+  INDEX test_case_dimension_results_result_id (testCaseResultId),
+  INDEX test_case_dimension_results_ts (timestamp)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+-- Add impersonatedBy column to all entity tables for tracking bot impersonation
+-- This column stores which bot performed an action on behalf of a user
+
+ALTER TABLE api_collection_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE api_endpoint_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE api_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE bot_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE chart_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE dashboard_data_model_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE dashboard_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE dashboard_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE data_contract_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE data_product_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE database_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE database_schema_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE dbservice_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE directory_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE domain_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE drive_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE event_subscription_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE file_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE glossary_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE glossary_term_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE ingestion_pipeline_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE kpi_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE messaging_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE metadata_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE metric_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE ml_model_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE mlmodel_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE notification_template_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE persona_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE pipeline_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE pipeline_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE policy_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE query_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE report_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE role_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE search_index_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE search_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE security_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE spreadsheet_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE storage_container_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE storage_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE stored_procedure_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE table_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE team_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE thread_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE topic_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE type_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE user_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE workflow_definition_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+ALTER TABLE worksheet_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
+
+UPDATE test_definition
+SET json = JSON_SET(
+    json,
+    '$.testPlatforms',
+    CAST(REPLACE(
+        JSON_EXTRACT(json, '$.testPlatforms'),
+        '"DBT"',
+        '"dbt"'
+    ) AS JSON)
+)
+WHERE JSON_CONTAINS(json, '"DBT"', '$.testPlatforms');
+
+-- Performance optimization for tag_usage prefix queries
+ALTER TABLE tag_usage
+ADD COLUMN targetfqnhash_lower VARCHAR(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
+GENERATED ALWAYS AS (CONVERT(LOWER(targetFQNHash) USING utf8mb4) COLLATE utf8mb4_unicode_ci) STORED;
+
+ALTER TABLE tag_usage
+ADD COLUMN tagfqn_lower VARCHAR(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
+GENERATED ALWAYS AS (CONVERT(LOWER(tagFQN) USING utf8mb4) COLLATE utf8mb4_unicode_ci) STORED;
+
+-- Create indexes on the new columns
+CREATE INDEX idx_targetfqnhash_lower ON tag_usage (targetfqnhash_lower(255));
+CREATE INDEX idx_tagfqn_lower ON tag_usage (tagfqn_lower(255));
+
+-- PRIMARY INDEX: For targetFQNHash prefix searches (LIKE 'prefix%')
+-- Using prefix indexing for large VARCHAR columns to stay within MySQL's 3072 byte limit
+CREATE INDEX idx_tag_usage_target_prefix_composite
+ON tag_usage (source, targetfqnhash_lower(255), state, tagFQN(255), labelType);
+
+-- For exact match queries
+CREATE INDEX idx_tag_usage_target_exact_composite
+ON tag_usage (source, targetFQNHash(255), state, tagFQN(255), labelType);
+
+-- For tagFQN prefix searches
+CREATE INDEX idx_tag_usage_tagfqn_prefix_composite
+ON tag_usage (source, tagfqn_lower(255), state, targetFQNHash(255), labelType);
+
+-- For JOIN operations
+CREATE INDEX idx_tag_usage_join_composite
+ON tag_usage (tagFQNHash(255), source, state, targetFQNHash(255), tagFQN(255), labelType);
+
+CREATE FULLTEXT INDEX ft_tag_usage_targetfqn
+ON tag_usage(targetFQNHash);
+
+-- Add index for efficient bulk term count queries
+-- The bulkGetTermCounts query uses: WHERE classificationHash IN (...) AND deleted = FALSE
+CREATE INDEX idx_tag_classification_deleted
+ON tag (classificationHash, deleted);
+
+-- Create new composite indexes with deleted column
+-- MySQL doesn't support INCLUDE clause, so we put all columns in the index
+CREATE INDEX idx_entity_relationship_from_deleted
+ON entity_relationship(fromId, fromEntity, relation, deleted, toId, toEntity);
+
+CREATE INDEX idx_entity_relationship_to_deleted
+ON entity_relationship(toId, toEntity, relation, deleted, fromId, fromEntity);
+
+-- Index for queries that filter by fromEntity/toEntity
+CREATE INDEX idx_entity_relationship_from_typed
+ON entity_relationship(toId, toEntity, relation, fromEntity, deleted, fromId);
+
+-- Index for bidirectional lookups
+CREATE INDEX idx_entity_relationship_bidirectional
+ON entity_relationship(fromId, toId, relation, deleted);
+
+
+

bootstrap/sql/migrations/native/1.11.0/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,108 @@
+-- Correct the table diff test definition
+-- This is to include the new parameter table2.keyColumns
+UPDATE test_definition
+SET json = json::jsonb || json_build_object(
+    'parameterDefinition', jsonb_build_array(
+        jsonb_build_object(
+            'name', 'keyColumns',
+            'displayName', 'Table 1''s key Columns',
+            'description', 'The columns to use as the key for the comparison. If not provided, it will be resolved from the primary key or unique columns. The tuples created from the key columns must be unique.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        jsonb_build_object(
+            'name', 'table2',
+            'displayName', 'Table 2',
+            'description', 'Fully qualified name of the table to compare against.',
+            'dataType', 'STRING',
+            'required', true
+        ),
+        jsonb_build_object(
+            'name', 'table2.keyColumns',
+            'displayName', 'Table 2''s key columns',
+            'description', 'The columns in table 2 to use as comparison. If not provided, it will default to `Key Columns`, risking errors if the key columns'' names have changed.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        jsonb_build_object(
+            'name', 'threshold',
+            'displayName', 'Threshold',
+            'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
+            'dataType', 'NUMBER',
+            'required', false
+        ),
+        jsonb_build_object(
+            'name', 'useColumns',
+            'displayName', 'Use Columns',
+            'description', 'Limits the scope of the test to this list of columns. If not provided, all columns will be used except the key columns.',
+            'dataType', 'ARRAY',
+            'required', false
+        ),
+        jsonb_build_object(
+            'name', 'where',
+            'displayName', 'SQL Where Clause',
+            'description', 'Use this where clause to filter the rows to compare.',
+            'dataType', 'STRING',
+            'required', false
+        ),
+        jsonb_build_object(
+            'name', 'caseSensitiveColumns',
+            'displayName', 'Case sensitive columns',
+            'description', 'Use case sensitivity when comparing the columns.',
+            'dataType', 'BOOLEAN',
+            'required', false
+        )
+    ),
+    'version', 0.2
+)::jsonb
+WHERE name = 'tableDiff';
+
+-- Update DataRetentionApplication add profileDataRetentionPeriod and testCaseResultsRetentionPeriod
+UPDATE installed_apps
+SET json = jsonb_set(
+    jsonb_set(
+        json,
+        '{appConfiguration, testCaseResultsRetentionPeriod}',
+        '1440'
+    ),
+    '{appConfiguration, profileDataRetentionPeriod}',
+    '1440'
+)
+WHERE json->>'name' = 'DataRetentionApplication';
+
+UPDATE notification_template_entity
+SET json = json::jsonb - 'defaultTemplateChecksum';
+
+-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAIQualityAgentApplication and CollateAITierAgentApplication
+UPDATE apps_marketplace
+SET json = jsonb_set(
+    jsonb_set(
+        json::jsonb,
+        '{appType}',
+        '"external"'
+    ),
+    '{sourcePythonClass}',
+    '"metadata.applications.dynamic_agent.app.DynamicAgentApp"'
+)
+WHERE json->>'name' IN  ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
+    AND json->>'appType' = 'internal' AND json->>'sourcePythonClass' IS NULL;
+
+-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAITierAgentApplication and CollateAIQualityAgentApplication
+UPDATE installed_apps
+SET json = jsonb_set(
+    jsonb_set(
+        json::jsonb,
+        '{appType}',
+        '"external"'
+    ),
+    '{sourcePythonClass}',
+    '"metadata.applications.dynamic_agent.app.DynamicAgentApp"'
+)
+WHERE json->>'name' IN ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
+  AND json->>'appType' = 'internal' AND json->>'sourcePythonClass' IS NULL;
+  
+-- Remove bot form App entity  
+UPDATE installed_apps SET json = json - 'bot';
+
+-- Remove SearchIndexingApplication past runs
+delete from apps_extension_time_series where appname = 'SearchIndexingApplication';

bootstrap/sql/migrations/native/1.11.0/postgres/schemaChanges.sql

@@ -0,0 +1,168 @@
+-- Test Case Dimension Results Time Series Table
+CREATE TABLE IF NOT EXISTS test_case_dimension_results_time_series (
+  entityFQNHash VARCHAR(768) COLLATE "C" NOT NULL,
+  extension VARCHAR(256) NOT NULL DEFAULT 'testCase.dimensionResult',
+  jsonSchema VARCHAR(256) NOT NULL,
+  json JSONB NOT NULL,
+  id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+  testCaseResultId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'testCaseResultId') STORED NOT NULL,
+  dimensionKey VARCHAR(512) GENERATED ALWAYS AS (json ->> 'dimensionKey') STORED NOT NULL,
+  dimensionName VARCHAR(256) GENERATED ALWAYS AS (SPLIT_PART(json ->> 'dimensionKey', '=', 1)) STORED,
+  timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
+  testCaseStatus VARCHAR(36) GENERATED ALWAYS AS (json ->> 'testCaseStatus') STORED,
+  CONSTRAINT test_case_dimension_results_unique_constraint UNIQUE (entityFQNHash, dimensionKey, timestamp)
+);
+
+-- Create indexes
+CREATE INDEX IF NOT EXISTS test_case_dimension_results_main ON test_case_dimension_results_time_series (entityFQNHash, timestamp, dimensionKey);
+CREATE INDEX IF NOT EXISTS test_case_dimension_results_dimension_name ON test_case_dimension_results_time_series (entityFQNHash, dimensionName, timestamp);
+CREATE INDEX IF NOT EXISTS test_case_dimension_results_result_id ON test_case_dimension_results_time_series (testCaseResultId);
+CREATE INDEX IF NOT EXISTS test_case_dimension_results_ts ON test_case_dimension_results_time_series (timestamp);
+-- Add impersonatedBy column to all entity tables for tracking bot impersonation
+-- This column stores which bot performed an action on behalf of a user
+
+ALTER TABLE api_collection_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE api_endpoint_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE api_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE bot_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE chart_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE dashboard_data_model_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE dashboard_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE dashboard_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE data_contract_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE data_product_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE database_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE database_schema_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE directory_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE domain_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE drive_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE event_subscription_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE file_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE glossary_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE glossary_term_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE ingestion_pipeline_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE kpi_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE messaging_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE metadata_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE metric_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE ml_model_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE mlmodel_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE notification_template_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE persona_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE pipeline_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE pipeline_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE policy_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE query_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE report_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE role_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE search_index_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE search_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE security_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE spreadsheet_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE storage_container_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE storage_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE stored_procedure_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE table_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE team_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE thread_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE topic_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE type_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE user_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE workflow_definition_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+ALTER TABLE worksheet_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{testPlatforms}',
+    REPLACE(
+        (json::jsonb -> 'testPlatforms')::text,
+        '"DBT"',
+        '"dbt"'
+    )::jsonb
+)::json
+WHERE json::jsonb -> 'testPlatforms' @> '"DBT"'::jsonb;
+
+-- Performance optimization for tag_usage prefix queries
+ALTER TABLE tag_usage
+ADD COLUMN IF NOT EXISTS targetfqnhash_lower text
+GENERATED ALWAYS AS (lower(targetFQNHash)) STORED;
+
+ALTER TABLE tag_usage
+ADD COLUMN IF NOT EXISTS tagfqn_lower text
+GENERATED ALWAYS AS (lower(tagFQN)) STORED;
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_target_prefix_covering
+ON tag_usage (source, targetfqnhash_lower text_pattern_ops)
+INCLUDE (tagFQN, labelType, state)
+WHERE state = 1;  -- Only active tags
+
+-- For exact match queries on targetFQNHash
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_target_exact
+ON tag_usage (source, targetFQNHash, state)
+INCLUDE (tagFQN, labelType);
+
+-- For tagFQN prefix searches if needed
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_tagfqn_prefix_covering
+ON tag_usage (source, tagfqn_lower text_pattern_ops)
+INCLUDE (targetFQNHash, labelType, state)
+WHERE state = 1;
+
+-- For JOIN operations with classification and tag tables
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_join_source
+ON tag_usage (tagFQNHash, source)
+INCLUDE (targetFQNHash, tagFQN, labelType, state)
+WHERE state = 1;
+
+-- Only create if you need %contains% searches
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- GIN index for substring matches (LIKE '%foo%')
+CREATE INDEX CONCURRENTLY IF NOT EXISTS gin_tag_usage_targetfqn_trgm
+ON tag_usage USING GIN (targetFQNHash gin_trgm_ops)
+WHERE state = 1;
+
+-- Optimize autovacuum for tag_usage (high update frequency)
+ALTER TABLE tag_usage SET (
+  autovacuum_vacuum_scale_factor = 0.05,    -- Vacuum at 5% dead rows (default 20%)
+  autovacuum_analyze_scale_factor = 0.02,   -- Analyze at 2% changed rows (default 10%)
+  autovacuum_vacuum_threshold = 50,         -- Minimum rows before vacuum
+  autovacuum_analyze_threshold = 50,        -- Minimum rows before analyze
+  fillfactor = 90                           -- Leave 10% free space for HOT updates
+);
+
+-- Increase statistics target for frequently queried columns
+ALTER TABLE tag_usage ALTER COLUMN targetFQNHash SET STATISTICS 1000;
+ALTER TABLE tag_usage ALTER COLUMN targetfqnhash_lower SET STATISTICS 1000;
+ALTER TABLE tag_usage ALTER COLUMN tagFQN SET STATISTICS 500;
+ALTER TABLE tag_usage ALTER COLUMN tagfqn_lower SET STATISTICS 500;
+ALTER TABLE tag_usage ALTER COLUMN source SET STATISTICS 100;
+
+-- Add index for efficient bulk term count queries
+-- The bulkGetTermCounts query uses: WHERE classificationHash IN (...) AND deleted = FALSE
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_classification_deleted
+ON tag (classificationHash, deleted);
+
+-- Create new indexes with deleted column for efficient filtering
+-- Using partial indexes (WHERE deleted = FALSE) for even better performance
+CREATE INDEX IF NOT EXISTS idx_entity_relationship_from_deleted
+ON entity_relationship(fromId, fromEntity, relation)
+INCLUDE (toId, toEntity, relation)
+WHERE deleted = FALSE;
+
+CREATE INDEX IF NOT EXISTS idx_entity_relationship_to_deleted
+ON entity_relationship(toId, toEntity, relation)
+INCLUDE (fromId, fromEntity, relation)
+WHERE deleted = FALSE;
+
+-- Also add indexes for the specific queries that include fromEntity/toEntity filters
+CREATE INDEX IF NOT EXISTS idx_entity_relationship_from_typed
+ON entity_relationship(toId, toEntity, relation, fromEntity)
+INCLUDE (fromEntity, toEntity)
+WHERE deleted = FALSE;
+
+-- Index for bidirectional lookups (used in UNION queries)
+CREATE INDEX IF NOT EXISTS idx_entity_relationship_bidirectional
+ON entity_relationship(fromId, toId, relation)
+WHERE deleted = FALSE;
+

bootstrap/sql/migrations/native/1.11.1/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+UPDATE
+    classification
+SET
+    json = JSON_SET(
+        json,
+        '$.autoClassificationConfig',
+        CAST(
+            '{"enabled": true, "conflictResolution": "highest_priority", "minimumConfidence": 0.6, "requireExplicitMatch": true}'
+            AS JSON
+        )
+    )
+WHERE
+    JSON_VALUE(json, '$.name' RETURNING CHAR) = 'PII';

bootstrap/sql/migrations/native/1.11.1/mysql/schemaChanges.sql

@@ -0,0 +1,61 @@
+-- Add Flowable batch processing tables for history cleanup support
+CREATE TABLE IF NOT EXISTS flw_ru_batch (
+    id_ VARCHAR(64) NOT NULL, 
+    rev_ INTEGER, 
+    type_ VARCHAR(64) NOT NULL, 
+    search_key_ VARCHAR(255), 
+    search_key2_ VARCHAR(255), 
+    create_time_ TIMESTAMP(3) NOT NULL, 
+    complete_time_ TIMESTAMP(3), 
+    status_ VARCHAR(255), 
+    batch_doc_id_ VARCHAR(64), 
+    tenant_id_ VARCHAR(255) DEFAULT '', 
+    CONSTRAINT pk_flw_ru_batch PRIMARY KEY (id_)
+);
+
+CREATE INDEX idx_flw_ru_batch_type ON flw_ru_batch (type_);
+CREATE INDEX idx_flw_ru_batch_search_key ON flw_ru_batch (search_key_);
+CREATE INDEX idx_flw_ru_batch_status ON flw_ru_batch (status_);
+
+CREATE TABLE IF NOT EXISTS flw_ru_batch_part (
+    id_ VARCHAR(64) NOT NULL, 
+    rev_ INTEGER, 
+    batch_id_ VARCHAR(64), 
+    type_ VARCHAR(64) NOT NULL, 
+    scope_id_ VARCHAR(64), 
+    sub_scope_id_ VARCHAR(64), 
+    scope_type_ VARCHAR(64), 
+    search_key_ VARCHAR(255), 
+    search_key2_ VARCHAR(255), 
+    create_time_ TIMESTAMP(3) NOT NULL, 
+    complete_time_ TIMESTAMP(3), 
+    status_ VARCHAR(255), 
+    result_doc_id_ VARCHAR(64), 
+    tenant_id_ VARCHAR(255) DEFAULT '', 
+    CONSTRAINT pk_flw_ru_batch_part PRIMARY KEY (id_)
+);
+
+CREATE INDEX idx_flw_ru_batch_part_batch_id ON flw_ru_batch_part (batch_id_);
+CREATE INDEX idx_flw_ru_batch_part_type ON flw_ru_batch_part (type_);
+CREATE INDEX idx_flw_ru_batch_part_status ON flw_ru_batch_part (status_);
+
+-- Update workflow settings with new history cleanup configuration fields
+UPDATE openmetadata_settings
+SET json = JSON_SET(
+    JSON_SET(
+        JSON_SET(
+            JSON_SET(
+                json,
+                '$.historyCleanUpConfiguration.batchSize',
+                1000
+            ),
+            '$.historyCleanUpConfiguration.timeCycleConfig',
+            '0 0 0 ? * 1'
+        ),
+        '$.runTimeCleanUpConfiguration',
+        JSON_OBJECT()
+    ),
+    '$.runTimeCleanUpConfiguration.batchSize',
+    500
+)
+WHERE configType = 'workflowSettings';

bootstrap/sql/migrations/native/1.11.1/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+UPDATE
+    classification
+SET
+    json = json::jsonb || json_build_object(
+        'autoClassificationConfig', json_build_object(
+            'enabled', true,
+            'conflictResolution', 'highest_priority',
+            'minimumConfidence', 0.6,
+            'requireExplicitMatch', true
+        )
+    )::jsonb
+WHERE
+    json->>'name' = 'PII';

bootstrap/sql/migrations/native/1.11.1/postgres/schemaChanges.sql

@@ -0,0 +1,65 @@
+-- Add Flowable batch processing tables for history cleanup support
+CREATE TABLE IF NOT EXISTS flw_ru_batch (
+    id_ VARCHAR(64) NOT NULL, 
+    rev_ INTEGER, 
+    type_ VARCHAR(64) NOT NULL, 
+    search_key_ VARCHAR(255), 
+    search_key2_ VARCHAR(255), 
+    create_time_ TIMESTAMP(3) NOT NULL, 
+    complete_time_ TIMESTAMP(3), 
+    status_ VARCHAR(255), 
+    batch_doc_id_ VARCHAR(64), 
+    tenant_id_ VARCHAR(255) DEFAULT '', 
+    CONSTRAINT pk_flw_ru_batch PRIMARY KEY (id_)
+);
+
+CREATE INDEX idx_flw_ru_batch_type ON flw_ru_batch (type_);
+CREATE INDEX idx_flw_ru_batch_search_key ON flw_ru_batch (search_key_);
+CREATE INDEX idx_flw_ru_batch_status ON flw_ru_batch (status_);
+
+CREATE TABLE IF NOT EXISTS flw_ru_batch_part (
+    id_ VARCHAR(64) NOT NULL, 
+    rev_ INTEGER, 
+    batch_id_ VARCHAR(64), 
+    type_ VARCHAR(64) NOT NULL, 
+    scope_id_ VARCHAR(64), 
+    sub_scope_id_ VARCHAR(64), 
+    scope_type_ VARCHAR(64), 
+    search_key_ VARCHAR(255), 
+    search_key2_ VARCHAR(255), 
+    create_time_ TIMESTAMP(3) NOT NULL, 
+    complete_time_ TIMESTAMP(3), 
+    status_ VARCHAR(255), 
+    result_doc_id_ VARCHAR(64), 
+    tenant_id_ VARCHAR(255) DEFAULT '', 
+    CONSTRAINT pk_flw_ru_batch_part PRIMARY KEY (id_)
+);
+
+CREATE INDEX idx_flw_ru_batch_part_batch_id ON flw_ru_batch_part (batch_id_);
+CREATE INDEX idx_flw_ru_batch_part_type ON flw_ru_batch_part (type_);
+CREATE INDEX idx_flw_ru_batch_part_status ON flw_ru_batch_part (status_);
+
+-- Update workflow settings with new history cleanup configuration fields
+UPDATE openmetadata_settings
+SET json = jsonb_set(
+    jsonb_set(
+        jsonb_set(
+            jsonb_set(
+                json,
+                '{historyCleanUpConfiguration,batchSize}',
+                '1000',
+                true
+            ),
+            '{historyCleanUpConfiguration,timeCycleConfig}',
+            '"0 0 0 ? * 1"',
+            true
+        ),
+        '{runTimeCleanUpConfiguration}',
+        '{}',
+        true
+    ),
+    '{runTimeCleanUpConfiguration,batchSize}',
+    '500',
+    true
+)
+WHERE configtype = 'workflowSettings';

bootstrap/sql/migrations/native/1.11.11/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+-- Migrate REST API service connections: move flat openAPISchemaURL into nested openAPISchemaConnection object
+UPDATE api_service_entity
+SET json = JSON_SET(
+    JSON_REMOVE(json, '$.connection.config.openAPISchemaURL'),
+    '$.connection.config.openAPISchemaConnection',
+    JSON_OBJECT(
+        'openAPISchemaURL',
+        JSON_UNQUOTE(JSON_EXTRACT(json, '$.connection.config.openAPISchemaURL'))
+    )
+)
+WHERE serviceType = 'Rest'
+  AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.openAPISchemaURL')
+  AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.openAPISchemaConnection');

bootstrap/sql/migrations/native/1.11.11/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,11 @@
+-- Migrate REST API service connections: move flat openAPISchemaURL into nested openAPISchemaConnection object
+UPDATE api_service_entity
+SET json = jsonb_set(
+    json #- '{connection,config,openAPISchemaURL}',
+    '{connection,config,openAPISchemaConnection}',
+    jsonb_build_object('openAPISchemaURL', json #> '{connection,config,openAPISchemaURL}'),
+    true
+)
+WHERE serviceType = 'Rest'
+  AND jsonb_exists(json -> 'connection' -> 'config', 'openAPISchemaURL')
+  AND NOT jsonb_exists(json -> 'connection' -> 'config', 'openAPISchemaConnection');

bootstrap/sql/migrations/native/1.11.12/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,26 @@
+-- Migrate OpenLineage connection config: move Kafka-specific fields into brokerConfig
+-- Supports the schema change adding Kinesis as an alternative broker type
+UPDATE pipeline_service_entity
+SET json = JSON_SET(
+    JSON_REMOVE(json,
+        '$.connection.config.brokersUrl',
+        '$.connection.config.topicName',
+        '$.connection.config.consumerGroupName',
+        '$.connection.config.consumerOffsets',
+        '$.connection.config.poolTimeout',
+        '$.connection.config.sessionTimeout',
+        '$.connection.config.securityProtocol',
+        '$.connection.config.sslConfig',
+        '$.connection.config.saslConfig'
+    ),
+    '$.connection.config.brokerConfig',
+    JSON_REMOVE(
+        JSON_EXTRACT(json, '$.connection.config'),
+        '$.type',
+        '$.pipelineFilterPattern',
+        '$.supportsMetadataExtraction'
+    )
+)
+WHERE serviceType = 'OpenLineage'
+  AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokersUrl')
+  AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokerConfig');

bootstrap/sql/migrations/native/1.11.12/mysql/schemaChanges.sql

@@ -0,0 +1,13 @@
+-- Migrate Redshift connection: move password into authType.password (basicAuth)
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.password'),
+    '$.connection.config.authType',
+    JSON_OBJECT(
+        'password',
+        JSON_EXTRACT(json, '$.connection.config.password')
+    )
+)
+WHERE serviceType = 'Redshift'
+  AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL
+  AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.authType');

bootstrap/sql/migrations/native/1.11.12/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,21 @@
+-- Migrate OpenLineage connection config: move Kafka-specific fields into brokerConfig
+-- Supports the schema change adding Kinesis as an alternative broker type
+UPDATE pipeline_service_entity
+SET json = jsonb_set(
+    json,
+    '{connection,config,brokerConfig}',
+    (json #> '{connection,config}') - 'type' - 'pipelineFilterPattern' - 'supportsMetadataExtraction',
+    true
+)
+#- '{connection,config,brokersUrl}'
+#- '{connection,config,topicName}'
+#- '{connection,config,consumerGroupName}'
+#- '{connection,config,consumerOffsets}'
+#- '{connection,config,poolTimeout}'
+#- '{connection,config,sessionTimeout}'
+#- '{connection,config,securityProtocol}'
+#- '{connection,config,sslConfig}'
+#- '{connection,config,saslConfig}'
+WHERE serviceType = 'OpenLineage'
+  AND jsonb_exists(json #> '{connection,config}', 'brokersUrl')
+  AND NOT jsonb_exists(json #> '{connection,config}', 'brokerConfig');

bootstrap/sql/migrations/native/1.11.12/postgres/schemaChanges.sql

@@ -0,0 +1,10 @@
+-- Migrate Redshift connection: move password into authType.password (basicAuth)
+UPDATE dbservice_entity
+SET json = jsonb_set(
+    json #- '{connection,config,password}',
+    '{connection,config,authType}',
+    jsonb_build_object('password', json #> '{connection,config,password}')
+)
+WHERE serviceType = 'Redshift'
+  AND json #> '{connection,config,password}' IS NOT NULL
+  AND NOT jsonb_exists(json #> '{connection,config}', 'authType');

bootstrap/sql/migrations/native/1.11.2/mysql/schemaChanges.sql

@@ -0,0 +1,3 @@
+-- No schema changes required for this migration
+-- This migration only fixes FQN hash for entities with service names containing dots
+-- The data migration is handled in Java code

bootstrap/sql/migrations/native/1.11.2/postgres/schemaChanges.sql

@@ -0,0 +1,3 @@
+-- No schema changes required for this migration
+-- This migration only fixes FQN hash for entities with service names containing dots
+-- The data migration is handled in Java code

bootstrap/sql/migrations/native/1.11.4/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,48 @@
+-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValueLengthsToBeBetween'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesMissingCount'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeInSet'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeNotInSet'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeNotNull'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeUnique'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToMatchRegex'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToNotMatchRegex'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');

bootstrap/sql/migrations/native/1.11.4/mysql/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- DO NOT RENAME OR DELETE THIS FILE
+-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1114