chore: import upstream snapshot with attribution

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v000__create_server_change_log.sql

@@ -0,0 +1,18 @@
+-- Create tables for tracking server migrations
+-- This migration runs before all other migrations to ensure migration tracking tables exist
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank SERIAL,
+    version VARCHAR(256) PRIMARY KEY,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    metrics JSONB
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v001__create_db_connection_info.sql

@@ -0,0 +1,430 @@
+CREATE OR REPLACE FUNCTION to_tz_timestamp(text) RETURNS TIMESTAMP WITH TIME ZONE AS
+$$
+select to_timestamp($1, '%Y-%m-%dT%T.%fZ')::timestamptz;
+$$
+    LANGUAGE sql immutable;
+
+--
+-- Table that captures all the relationships between entities
+--
+CREATE TABLE IF NOT EXISTS entity_relationship (
+    fromId VARCHAR(36) NOT NULL,                -- ID of the from entity
+    toId VARCHAR(36) NOT NULL,                  -- ID of the to entity
+    fromEntity VARCHAR(256) NOT NULL,           -- Type name of the from entity
+    toEntity VARCHAR(256) NOT NULL,             -- Type name of to entity
+    relation SMALLINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSONB,                                 -- JSON payload with additional information
+    deleted BOOLEAN NOT NULL DEFAULT FALSE,
+    PRIMARY KEY (fromId, toId, relation)
+);
+
+CREATE INDEX IF NOT EXISTS entity_relationship_edge_index ON entity_relationship(fromId, toId, relation);
+CREATE INDEX IF NOT EXISTS entity_relationship_from_index ON entity_relationship(fromId, relation);
+CREATE INDEX IF NOT EXISTS entity_relationship_to_index ON entity_relationship(toId, relation);
+
+--
+-- Table that captures all the relationships between field of an entity to a field of another entity
+-- Example - table1.column1 (fromFQN of type table.columns.column) is joined with(relation)
+-- table2.column8 (toFQN of type table.columns.column)
+--
+CREATE TABLE IF NOT EXISTS field_relationship (
+    fromFQN VARCHAR(256) NOT NULL,              -- Fully qualified name of entity or field
+    toFQN VARCHAR(256) NOT NULL,                -- Fully qualified name of entity or field
+    fromType VARCHAR(256) NOT NULL,             -- Fully qualified type of entity or field
+    toType VARCHAR(256) NOT NULL,               -- Fully qualified type of entity or field
+    relation SMALLINT NOT NULL,
+    jsonSchema VARCHAR(256),                    -- Schema used for generating JSON
+    json JSONB,                                 -- JSON payload with additional information
+    PRIMARY KEY (fromFQN, toFQN, relation)
+);
+
+CREATE INDEX IF NOT EXISTS field_relationship_from_index ON field_relationship(fromFQN, relation);
+CREATE INDEX IF NOT EXISTS field_relationship_to_index ON field_relationship(toFQN, relation);
+
+--
+-- Used for storing additional metadata for an entity
+--
+CREATE TABLE IF NOT EXISTS entity_extension (
+    id VARCHAR(36) NOT NULL,                    -- ID of the from entity
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSONB NOT NULL,
+    PRIMARY KEY (id, extension)
+);
+
+--
+-- Service entities
+--
+CREATE TABLE IF NOT EXISTS dbservice_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS messaging_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS storage_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Data entities
+--
+CREATE TABLE IF NOT EXISTS database_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS database_schema_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt') :: bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted') :: boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS table_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS metric_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS report_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS dashboard_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS ml_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS topic_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS chart_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS location_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Feed related tables
+--
+CREATE TABLE IF NOT EXISTS thread_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityId') STORED NOT NULL,
+    entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'about') STORED NOT NULL,
+    assignedTo VARCHAR(256) GENERATED ALWAYS AS (json ->> 'addressedTo') STORED,
+    json JSONB NOT NULL,
+    createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'threadTs')::bigint) STORED NOT NULL,
+    createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    resolved BOOLEAN GENERATED ALWAYS AS ((json ->> 'resolved')::boolean) STORED,
+    PRIMARY KEY (id)
+);
+
+--
+-- Policies related tables
+--
+CREATE TABLE IF NOT EXISTS policy_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- Ingestion related tables
+--
+CREATE TABLE IF NOT EXISTS ingestion_pipeline_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSON NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    timestamp BIGINT,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+--
+-- User, Team, and bots
+--
+CREATE TABLE IF NOT EXISTS team_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS user_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    email VARCHAR(256) GENERATED ALWAYS AS (json ->> 'email') STORED NOT NULL,
+    deactivated VARCHAR(8) GENERATED ALWAYS AS (json ->> 'deactivated') STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS bot_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS role_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    defaultRole BOOLEAN GENERATED ALWAYS AS ((json ->> 'defaultRole')::boolean) STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+--
+-- Usage table where usage for all the entities is captured
+--
+CREATE TABLE IF NOT EXISTS entity_usage (
+    id VARCHAR(36) NOT NULL,         -- Unique id of the entity
+    entityType VARCHAR(20) NOT NULL, -- name of the entity for which this usage is published
+    usageDate DATE,                  -- date corresponding to the usage
+    count1 INT,                      -- total daily count of use on usageDate
+    count7 INT,                      -- rolling count of last 7 days going back from usageDate
+    count30 INT,                     -- rolling count of last 30 days going back from usageDate
+    percentile1 INT,                 -- percentile rank with in same entity for given usage date
+    percentile7 INT,                 -- percentile rank with in same entity for last 7 days of usage
+    percentile30 INT,                -- percentile rank with in same entity for last 30 days of usage
+    UNIQUE (usageDate, id)
+);
+
+--
+-- Tag related tables
+--
+CREATE TABLE IF NOT EXISTS tag_category (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL, -- JSON stores category information and does not store children
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name) -- Unique tag category name
+);
+
+CREATE TABLE IF NOT EXISTS tag (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL, -- JSON stores all tag attributes and does not store children
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS tag_usage (
+    source SMALLINT NOT NULL,           -- Source of the tag label
+    tagFQN VARCHAR(256) NOT NULL,       -- Fully qualified name of the tag
+    targetFQN VARCHAR(256) NOT NULL,    -- Fully qualified name of the entity instance or corresponding field
+    labelType SMALLINT NOT NULL,        -- Type of tagging: manual, automated, propagated, derived
+    state SMALLINT NOT NULL,            -- State of tagging: suggested or confirmed
+    UNIQUE (source, tagFQN, targetFQN)
+);
+
+CREATE TABLE IF NOT EXISTS change_event (
+    eventType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
+    userName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'userName') STORED NOT NULL,
+    eventTime BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
+    json JSONB NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS change_event_event_type_index ON change_event(eventType);
+CREATE INDEX IF NOT EXISTS change_event_entity_type_index ON change_event(entityType);
+CREATE INDEX IF NOT EXISTS change_event_event_time_index ON change_event(eventTime);
+
+CREATE TABLE IF NOT EXISTS webhook_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSON NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+    -- No versioning, updatedAt, updatedBy, or changeDescription fields for webhook
+);
+
+CREATE TABLE IF NOT EXISTS glossary_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS glossary_term_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v002__create_db_connection_info.sql

@@ -0,0 +1,78 @@
+CREATE TABLE IF NOT EXISTS type_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    category VARCHAR(256) GENERATED ALWAYS AS (json ->> 'category') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+ALTER TABLE webhook_entity
+ADD status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED NOT NULL,
+DROP COLUMN deleted;
+
+DROP INDEX entity_relationship_edge_index;
+
+CREATE TABLE IF NOT EXISTS mlmodel_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE thread_entity
+SET json = jsonb_set(json, '{type}', '"Conversation"', true);
+
+update thread_entity
+SET json = jsonb_set(json, '{reactions}', '[]'::jsonb, true);
+
+ALTER TABLE thread_entity
+    ADD type VARCHAR(64) GENERATED ALWAYS AS (json ->> 'type') STORED NOT NULL,
+    ADD taskId INT GENERATED ALWAYS AS ((json#>'{task,id}')::integer) STORED,
+    ADD taskStatus VARCHAR(64) GENERATED ALWAYS AS (json#>>'{task,status}') STORED,
+    ADD taskAssignees JSONB GENERATED ALWAYS AS (json#>'{task,assignees}') STORED,
+    ADD CONSTRAINT task_id_constraint UNIQUE(taskId);
+
+CREATE INDEX IF NOT EXISTS thread_entity_type_index ON thread_entity(type);
+CREATE INDEX IF NOT EXISTS thread_entity_task_assignees_index ON thread_entity(taskAssignees);
+CREATE INDEX IF NOT EXISTS thread_entity_task_status_index ON thread_entity(taskStatus);
+CREATE INDEX IF NOT EXISTS thread_entity_created_by_index ON thread_entity(createdBy);
+CREATE INDEX IF NOT EXISTS thread_entity_updated_at_index ON thread_entity(updatedAt);
+
+CREATE TABLE task_sequence (id SERIAL PRIMARY KEY, dummy varchar(1));
+INSERT INTO task_sequence (dummy) VALUES (0) RETURNING id;
+
+DELETE from ingestion_pipeline_entity where 1=1;
+DELETE FROM pipeline_service_entity WHERE 1=1;
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,databaseSchema}', json#>'{connection,config,database}')
+where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle')
+  and json#>'{connection,config,database}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+where serviceType in ('Mysql','Hive','Presto','Trino','Clickhouse','SingleStore','MariaDB','Db2','Oracle');
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}' #- '{connection,config,username}' #- '{connection,config,projectId}' #- '{connection,config,enablePolicyTagImport}'
+WHERE serviceType = 'BigQuery';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+WHERE serviceType in ('Athena','Databricks');
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}' #- '{connection,config,pipelineServiceName}'
+WHERE serviceType = 'Glue';
+
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,dbServiceName}'
+WHERE serviceType in ('Metabase','Superset','Tableau');

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v003__create_db_connection_info.sql

@@ -0,0 +1,31 @@
+DELETE from ingestion_pipeline_entity where 1=1;
+DELETE from entity_relationship where toEntity = 'ingestionPipeline';
+
+-- 0.10 had empty FQNs for services, users and teams
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE messaging_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE storage_service_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE team_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;
+
+UPDATE user_entity
+SET json = jsonb_set(json, '{fullyQualifiedName}', json#>'{name}')
+WHERE json#>'{fullyQualifiedName}' is NULL;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v004__create_db_connection_info.sql

@@ -0,0 +1,113 @@
+UPDATE team_entity
+SET json = JSONB_SET(json, '{teamType}', '"Group"', true);
+
+ALTER TABLE team_entity
+ADD teamType VARCHAR(64) GENERATED ALWAYS AS (json ->> 'teamType') STORED NOT NULL;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,database}'
+where serviceType = 'DynamoDB';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,connectionOptions}'
+where serviceType = 'DeltaLake';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}'
+where serviceType = 'DeltaLake';
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{connection,config,clientId}', json#>'{connection,config,username}')
+WHERE serviceType = 'Looker'
+  and json#>'{connection,config,username}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{connection,config,clientSecret}', json#>'{connection,config,password}')
+WHERE serviceType = 'Looker'
+  and json#>'{connection,config,password}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}' #- '{connection,config,env}'
+WHERE serviceType = 'Looker';
+
+CREATE TABLE IF NOT EXISTS test_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'entityType') STORED NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_suite (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS test_case (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(512) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    entityFQN VARCHAR (712) GENERATED ALWAYS AS (json ->> 'entityFQN') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE webhook_entity
+SET json = JSONB_SET(json::jsonb, '{webhookType}', '"generic"', true);
+
+ALTER TABLE webhook_entity
+    ADD webhookType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'webhookType') STORED NOT NULL;
+
+CREATE TABLE IF NOT EXISTS entity_extension_time_series (
+    entityFQN VARCHAR(1024) NOT NULL,           -- Entity FQN, we can refer to tables and columns
+    extension VARCHAR(256) NOT NULL,            -- Extension name same as entity.fieldName
+    jsonSchema VARCHAR(256) NOT NULL,           -- Schema used for generating JSON
+    json JSONB NOT NULL,
+    timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
+);
+
+
+ALTER TABLE thread_entity
+    ADD announcementStart BIGINT GENERATED ALWAYS AS ((json#>'{announcement,startTime}')::bigint) STORED,
+    ADD announcementEnd BIGINT GENERATED ALWAYS AS ((json#>'{announcement,endTime}')::bigint) STORED;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,databaseSchema}' #- '{connection,config,oracleServiceName}'
+WHERE serviceType = 'Oracle';
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,hostPort}'
+WHERE serviceType = 'Athena';
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,username}' #- '{connection,config,password}'
+WHERE serviceType in ('Databricks');
+
+CREATE TABLE IF NOT EXISTS openmetadata_settings (
+    id SERIAL NOT NULL ,
+    configType VARCHAR(36) NOT NULL,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id, configType),
+    UNIQUE(configType)
+);
+
+DELETE FROM entity_extension
+WHERE jsonSchema IN ('tableProfile', 'columnTest', 'tableTest');
+
+DELETE FROM ingestion_pipeline_entity 
+WHERE json_extract_path_text("json", 'pipelineType') = 'profiler';
+
+DELETE FROM role_entity;
+DELETE FROM policy_entity;
+DELETE FROM field_relationship WHERE fromType IN ('role', 'policy') OR toType IN ('role', 'policy');
+DELETE FROM entity_relationship WHERE fromEntity IN ('role', 'policy') OR toEntity IN ('role', 'policy');
+ALTER TABLE role_entity DROP COLUMN defaultRole;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v005__create_db_connection_info.sql

@@ -0,0 +1,347 @@
+DELETE FROM entity_relationship
+WHERE toEntity = 'ingestionPipeline'
+AND toId NOT IN (
+	SELECT DISTINCT id 
+	FROM ingestion_pipeline_entity
+);
+
+CREATE TABLE IF NOT EXISTS user_tokens (
+    token VARCHAR(36) GENERATED ALWAYS AS (json ->> 'token') STORED NOT NULL,
+    userId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'userId') STORED NOT NULL,
+    tokenType VARCHAR(50) GENERATED ALWAYS AS (json ->> 'tokenType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    expiryDate BIGINT GENERATED ALWAYS AS ((json ->> 'expiryDate')::bigint) STORED NOT NULL,
+    PRIMARY KEY (token)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(
+        json,
+        '{connection,config,metastoreConnection}',
+        jsonb_build_object('metastoreHostPort', json#>'{connection,config,metastoreHostPort}')
+    )
+WHERE serviceType = 'DeltaLake'
+  AND json#>'{connection,config,metastoreHostPort}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,metastoreHostPort}'
+WHERE serviceType = 'DeltaLake';
+
+UPDATE dbservice_entity
+SET json = jsonb_set(
+        json,
+        '{connection,config,metastoreConnection}',
+        jsonb_build_object('metastoreFilePath', json#>'{connection,config,metastoreFilePath}')
+    )
+WHERE serviceType = 'DeltaLake'
+  AND json#>'{connection,config,metastoreFilePath}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,metastoreFilePath}'
+WHERE serviceType = 'DeltaLake';
+
+ALTER TABLE test_definition 
+ADD supported_data_types JSONB GENERATED ALWAYS AS (json -> 'supportedDataTypes') STORED;
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnCountToEqual"',
+  false
+)
+WHERE json->>'name' = 'TableColumnCountToEqual';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnCountToEqual"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnCountToEqual';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnToMatchSet"',
+  false
+)
+WHERE json->>'name' = 'TableColumnToMatchSet';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnToMatchSet"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnToMatchSet';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnNameToExist"',
+  false
+)
+WHERE json->>'name' = 'TableColumnNameToExist';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnNameToExist"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnNameToExist';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableRowCountToBeBetween"',
+  false
+)
+WHERE json->>'name' = 'TableRowCountToBeBetween';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableRowCountToBeBetween"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableRowCountToBeBetween';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableColumnCountToBeBetween"',
+  false
+)
+WHERE json->>'name' = 'TableColumnCountToBeBetween';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableColumnCountToBeBetween"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableColumnCountToBeBetween';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"columnValuesToBeInSet"',
+  false
+)
+WHERE json->>'name' = 'ColumnValuesToBeInSet';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"columnValuesToBeInSet"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'ColumnValuesToBeInSet';
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{name}',
+  '"tableRowCountToEqual"',
+  false
+)
+WHERE json->>'name' = 'TableRowCountToEqual';
+
+UPDATE test_definition 
+SET json = jsonb_set(
+  json,
+  '{fullyQualifiedName}',
+  '"tableRowCountToEqual"',
+  false
+)
+WHERE json->>'fullyQualifiedName' = 'TableRowCountToEqual';
+
+
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMeanToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "ARRAY", "SET"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMaxToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMedianToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueMinToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueLengthsToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesMissingCount'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesSumToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValueStdDevToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeBetween'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeNotInSet'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeNotNull'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToBeUnique'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"]',
+  true
+) 
+WHERE json->>'name' = 'columnValuesToNotMatchRegex'
+AND supported_data_types IS NULL;
+
+UPDATE pipeline_service_entity
+SET json = json::jsonb #- '{connection,config,dbConnection}'
+WHERE serviceType = 'Dagster';
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(
+      jsonb_set(
+        json,
+        '{serviceType}',
+        '"GluePipeline"',
+        true
+      ),
+      '{connection,config,type}',
+      '"GluePipeline"',
+      true
+  )
+WHERE serviceType = 'Glue';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v006__create_db_connection_info.sql

@@ -0,0 +1,155 @@
+--
+-- Upgrade changes for 0.13
+--
+
+
+ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQN TYPE varchar(768);
+CREATE INDEX IF NOT EXISTS entity_extension_time_series_entity_fqn_index ON entity_extension_time_series(entityFQN);
+
+CREATE TABLE IF NOT EXISTS web_analytic_event (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    eventType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'eventType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
+
+UPDATE bot_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
+
+UPDATE bot_entity
+SET json = json::jsonb #- '{botType}';
+
+CREATE TABLE IF NOT EXISTS data_insight_chart (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    dataIndexType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'dataIndexType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
+
+UPDATE role_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE name in ('DataConsumer', 'DataSteward');
+
+UPDATE policy_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE fullyQualifiedName in ('DataConsumerPolicy', 'DataStewardPolicy', 'OrganizationPolicy', 'TeamOnlyPolicy');
+
+UPDATE tag_category
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true)
+WHERE fullyQualifiedName in ('PersonalData.Personal', 'PersonalData.SpecialCategory',
+'PII.None', 'PII.NonSensitive', 'PII.Sensitive',
+'Tier.Tier1', 'Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5');
+
+UPDATE pipeline_service_entity 
+SET json = JSONB_SET(json::jsonb,'{connection,config}',json::jsonb #>'{connection,config}' || jsonb_build_object('configSource',jsonb_build_object('hostPort',json #>'{connection,config,hostPort}')), true)
+where servicetype = 'Dagster';
+
+UPDATE pipeline_service_entity 
+SET json = json::jsonb #- '{connection,config,hostPort}' #- '{connection,config,numberOfStatus}'
+where servicetype = 'Dagster';
+
+-- Remove categoryType
+UPDATE tag_category
+SET json = json::jsonb #- '{categoryType}';
+
+-- set mutuallyExclusive flag
+UPDATE tag_category
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
+
+UPDATE tag_category
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'true'::jsonb, true)
+WHERE name in ('PersonalData', 'PII', 'Tier');
+
+UPDATE tag
+SET json = jsonb_set(json, '{mutuallyExclusive}', 'false'::jsonb, true);
+
+-- Merge mutually exclusive table when multiple tag labels are used
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier1' AND
+t1.tagFQN in ('Tier.Tier2', 'Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier2' AND
+t1.tagFQN in ('Tier.Tier3', 'Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier3' AND
+t1.tagFQN in ('Tier.Tier4', 'Tier.Tier5')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'Tier.Tier4' AND
+t1.tagFQN in ('Tier.Tier5'));
+
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PII.Sensitive' AND
+t1.tagFQN in ('PII.NonSensitive', 'PII.None')
+) OR
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PII.NonSensitive' AND
+t1.tagFQN in ('PII.None'));
+
+DELETE FROM tag_usage t1
+USING tag_usage t2
+WHERE
+(t1.targetFQN = t2.targetFQN AND
+t2.tagFQN = 'PersonalData.Personal' AND
+t1.tagFQN in ('PersonalData.SpecialCategory'));
+
+CREATE TABLE IF NOT EXISTS kpi_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS metadata_service_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- We are starting to store the current deployed flag. Let's mark it as false by default
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{deployed}';
+
+UPDATE ingestion_pipeline_entity
+SET json = jsonb_set(json::jsonb, '{deployed}', 'true'::jsonb, true);
+
+-- We removed the supportsMetadataExtraction field in the `OpenMetadataConnection` object being used in IngestionPipelines
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{openMetadataServerConnection,supportsMetadataExtraction}';
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v007__create_db_connection_info.sql

@@ -0,0 +1,63 @@
+-- Remove markDeletedTablesFromFilterOnly 
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,markDeletedTablesFromFilterOnly}';
+
+UPDATE data_insight_chart
+SET json = jsonb_set(
+        json,
+        '{dimensions}',
+        '[{"name":"entityFqn","chartDataType":"STRING"},{"name":"entityType","chartDataType":"STRING"},{"name":"owner","chartDataType":"STRING"},{"name":"entityHref","chartDataType":"STRING"}]'
+)
+WHERE name = 'mostViewedEntities';
+
+DROP TABLE webhook_entity;
+
+CREATE TABLE IF NOT EXISTS alert_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS alert_action_def (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    alertActionType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'alertActionType') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,database}', json#>'{connection,config,databaseSchema}')
+where serviceType in ('Db2')
+  and json#>'{connection,config,databaseSchema}' is not null;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,databaseSchema}'
+where serviceType in ('Db2');
+
+DELETE from openmetadata_settings where configType = 'activityFeedFilterSetting';
+
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource}';
+
+UPDATE pipeline_service_entity 
+SET json = jsonb_set(jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,token}', json#> '{connection,config,configSource,token}', true) ,'{connection,config,host}', json #> '{connection,config,configSource,host}' , true)
+WHERE serviceType = 'Dagster' and json #>'{connection,config,configSource,host}' is not null;
+
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,host}', json#> '{connection,config,configSource,hostPort}', true)
+WHERE servicetype = 'Dagster' and json #>'{connection,config,configSource,hostPort}' is not null;
+
+UPDATE topic_entity
+SET json = jsonb_set(json::jsonb #- '{schemaText}', '{messageSchema}', jsonb_build_object('schemaText', json#>'{schemaText}'), true)
+WHERE json #> '{schemaText}' IS NOT NULL;
+
+UPDATE topic_entity
+SET json = jsonb_set(json::jsonb #- '{schemaType}', '{messageSchema,schemaType}', json#> '{schemaType}', true)
+WHERE json #> '{schemaType}' IS NOT NULL;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v008__create_db_connection_info.sql

@@ -0,0 +1,53 @@
+ALTER TABLE tag_category
+RENAME TO classification;
+
+-- Rename tagCategoryName in BigQuery for classificationName
+UPDATE dbservice_entity
+SET json = jsonb_set(json, '{connection,config,classificationName}', json#>'{connection,config,tagCategoryName}')
+where serviceType in ('BigQuery')
+  and json#>'{connection,config,tagCategoryName}' is not null;
+
+-- Delete supportsUsageExtraction from vertica
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsUsageExtraction}'
+WHERE serviceType = 'Vertica';
+
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtUpdateDescriptions}'
+WHERE json#>>'{sourceConfig,config,type}' = 'DBT';
+
+UPDATE test_definition
+SET json = jsonb_set(
+  json,
+  '{supportedDataTypes}',
+  '["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "TIMESTAMP", "TIMESTAMPZ","DATETIME", "DATE"]',
+  false
+)
+WHERE json->>'name' = 'columnValuesToBeBetween';
+
+UPDATE pipeline_entity
+SET json = jsonb_set(
+        json,
+        '{name}',
+        to_jsonb(replace(json ->> 'name',':',''))
+    )
+WHERE json ->> 'serviceType' = 'Dagster';
+
+UPDATE pipeline_entity
+SET json = jsonb_set(
+        json,
+        '{fullyQualifiedName}',
+        to_jsonb(replace(json ->> 'fullyQualifiedName',':',''))
+    )
+WHERE json ->> 'serviceType' = 'Dagster';
+
+UPDATE dashboard_service_entity  
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}' #- '{provider}'|| 
+jsonb_build_object('connection',jsonb_build_object(
+'username',json #>'{connection,config,username}',
+'password',json #>'{connection,config,password}',
+'provider',json #>'{connection,config,provider}'
+)), true)
+where servicetype = 'Superset';
+

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v009__create_db_connection_info.sql

@@ -0,0 +1,231 @@
+-- Unique constraint for user email address
+ALTER TABLE user_entity
+ADD UNIQUE (email);
+
+
+-- Remove classificationName in BigQuery
+UPDATE dbservice_entity SET json = json #- '{connection,config,classificationName}' where serviceType in ('BigQuery');
+
+-- migrate ingestAllDatabases in postgres 
+UPDATE dbservice_entity de2 
+SET json = JSONB_SET(
+    json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'|| 
+    jsonb_build_object('database',
+    (SELECT json->>'name' 
+        FROM database_entity de 
+        WHERE id = (SELECT er.toId 
+                    FROM entity_relationship er 
+                    WHERE er.fromId = de2.id 
+                    AND er.toEntity = 'database' 
+                    LIMIT 1)
+    )
+    )), 
+    '{connection,config,ingestAllDatabases}', 
+    'true'::jsonb
+)
+WHERE de2.serviceType = 'Postgres' 
+AND json->>'{connection,config,database}' IS NULL;
+
+CREATE TABLE IF NOT EXISTS storage_container_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+CREATE TABLE IF NOT EXISTS test_connection_definition (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    UNIQUE (name)
+);
+
+CREATE TABLE IF NOT EXISTS automations_workflow (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'workflowType') STORED NOT NULL,
+    status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+-- Do not store OM server connection, we'll set it dynamically on the resource
+UPDATE ingestion_pipeline_entity
+SET json = json::jsonb #- '{openMetadataServerConnection}';
+
+CREATE TABLE IF NOT EXISTS query_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    PRIMARY KEY (id),
+    UNIQUE (name)  
+);
+
+CREATE TABLE IF NOT EXISTS temp_query_migration (
+    tableId VARCHAR(36) NOT NULL,
+    queryId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    queryName VARCHAR(255) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    json JSONB NOT NULL
+);
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+INSERT INTO temp_query_migration(tableId,json)
+SELECT id,json_build_object('id',gen_random_uuid(),'query',query,'users',users,'checksum',checksum,'duration',duration,'name',checksum,'updatedAt',
+floor(EXTRACT(EPOCH FROM NOW())),'updatedBy','admin','deleted',false) AS json FROM entity_extension AS ee , jsonb_to_recordset(ee.json) AS x (query varchar,users json,
+checksum varchar,name varchar, duration decimal,queryDate varchar)
+WHERE ee.extension = 'table.tableQueries';
+
+INSERT INTO query_entity (json)
+SELECT value
+FROM (
+  SELECT jsonb_object_agg(queryName, json) AS json_data FROM ( SELECT DISTINCT queryName, json FROM temp_query_migration) subquery
+) cte, jsonb_each(cte.json_data)
+ON CONFLICT (name) DO UPDATE SET json = EXCLUDED.json;
+
+INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation)
+SELECT tmq.tableId, qe.id, 'table', 'query', 5
+FROM temp_query_migration tmq
+JOIN query_entity qe ON qe.name = tmq.queryName;
+
+DELETE FROM entity_extension WHERE id in
+(SELECT DISTINCT tableId FROM temp_query_migration) AND extension = 'table.tableQueries';
+
+DROP TABLE temp_query_migration;
+
+-- remove the audience if it was wrongfully sent from the UI after editing the OM service
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{connection,config,securityConfig,audience}'
+WHERE name = 'OpenMetadata'
+    AND json#>'{connection,config,authProvider}' IS NOT NULL
+    AND json -> 'connection' -> 'config' ->> 'authProvider' != 'google';
+
+ALTER TABLE user_tokens ALTER COLUMN expiryDate DROP NOT NULL;
+
+CREATE TABLE IF NOT EXISTS event_subscription_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    json JSONB NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (name)
+);
+
+drop table if exists alert_action_def;
+drop table if exists alert_entity;
+DELETE from entity_relationship where  fromEntity = 'alert' and toEntity = 'alertAction';
+
+-- create data model table
+CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
+    id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
+    json JSONB NOT NULL,
+    updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
+    updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
+    deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
+    fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+    PRIMARY KEY (id),
+    UNIQUE (fullyQualifiedName)
+);
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,database}', '{connection,config,databaseName}', json#> '{connection,config,database}', true)
+WHERE servicetype = 'Druid' and json #>'{connection,config,database}' is not null;
+
+-- We were using the same jsonSchema for Pipeline Services and Ingestion Pipeline status
+-- Also, we relied on the extension to store the run id
+UPDATE entity_extension_time_series
+SET jsonSchema = 'ingestionPipelineStatus', extension = 'ingestionPipeline.pipelineStatus'
+WHERE jsonSchema = 'pipelineStatus' AND extension <> 'pipeline.PipelineStatus';
+
+-- We are refactoring the storage service with containers. We'll remove the locations
+DROP TABLE location_entity;
+DELETE FROM entity_relationship WHERE fromEntity='location' OR toEntity='location';
+TRUNCATE TABLE storage_service_entity;
+
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,storageServiceName}'
+WHERE servicetype = 'Glue';
+
+UPDATE chart_entity
+SET json = json::jsonb #- '{tables}';
+
+-- Updating the tableau authentication fields
+UPDATE dashboard_service_entity
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{password}' #- '{username}'|| 
+jsonb_build_object('authType',jsonb_build_object(
+'username',json #>'{connection,config,username}',
+'password',json #>'{connection,config,password}'
+)), true)
+where servicetype = 'Tableau'
+and json#>'{connection,config,password}' is not null
+and json#>'{connection,config,username}' is not null;
+
+UPDATE dashboard_service_entity
+SET json = JSONB_SET(json::jsonb,
+'{connection,config}',json::jsonb #>'{connection,config}' #- '{personalAccessTokenName}' #- '{personalAccessTokenSecret}'|| 
+jsonb_build_object('authType',jsonb_build_object(
+'personalAccessTokenName',json #>'{connection,config,personalAccessTokenName}',
+'personalAccessTokenSecret',json #>'{connection,config,personalAccessTokenSecret}'
+)), true)
+where servicetype = 'Tableau'
+and json#>'{connection,config,personalAccessTokenName}' is not null
+and json#>'{connection,config,personalAccessTokenSecret}' is not null;
+
+-- Removed property from metadataService.json
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{allowServiceCreation}'
+WHERE serviceType in ('Amundsen', 'Atlas', 'MetadataES', 'OpenMetadata');
+
+UPDATE metadata_service_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"')
+WHERE name = 'OpenMetadata';
+
+-- Fix Glue sample data endpoint URL to be a correct URI
+UPDATE dbservice_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,awsConfig,endPointURL}', '"https://glue.region_name.amazonaws.com/"')
+WHERE serviceType = 'Glue'
+  AND json#>'{connection,config,awsConfig,endPointURL}' = '"https://glue.<region_name>.amazonaws.com/"';
+
+-- Delete connectionOptions from superset
+UPDATE dashboard_service_entity
+SET json = json::jsonb #- '{connection,config,connectionOptions}'
+WHERE serviceType = 'Superset';
+
+-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,partitionQueryDuration}' #- '{connection,config,partitionQuery}' #- '{connection,config,partitionField}'
+WHERE serviceType = 'BigQuery';
+
+-- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsQueryComment}' #- '{connection,config,scheme}' #- '{connection,config,hostPort}' #- '{connection,config,supportsProfiler}'
+WHERE serviceType = 'Salesforce';
+
+-- Delete supportsProfiler from DynamoDB
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,supportsProfiler}'
+WHERE serviceType = 'DynamoDB';
+
+-- Update TagLabels source from 'Tag' to 'Classification' after #10486
+UPDATE table_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
+UPDATE ml_model_entity SET json = REGEXP_REPLACE(json::text, '"source"\s*:\s*"Tag\"', '"source": "Classification"', 'g')::jsonb;
+
+-- Delete uriString from Mssql
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,uriString}'
+WHERE serviceType = 'Mssql';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v010__create_db_connection_info.sql

@@ -0,0 +1,17 @@
+-- Updating the value of SASL Mechanism for Kafka and Redpanda connections
+UPDATE messaging_service_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,saslMechanism}', '"PLAIN"')
+WHERE (servicetype = 'Kafka' OR serviceType = 'Redpanda')
+  AND json#>'{connection,config,saslMechanism}' IS NOT NULL
+  AND json#>'{connection,config,saslMechanism}' NOT IN ('"GSSAPI"', '"PLAIN"', '"SCRAM-SHA-256"', '"SCRAM-SHA-512"', '"OAUTHBEARER"');
+
+-- Remove the Subscriptions
+DELETE FROM event_subscription_entity;
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove the ibmi scheme from Db2 replace it with db2+ibm_db
+UPDATE dbservice_entity
+SET json = JSONB_SET(json::jsonb, '{connection,config,scheme}', '"db2+ibm_db"')
+WHERE serviceType  = 'Db2';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v011__create_db_connection_info.sql

@@ -0,0 +1,4 @@
+-- Update the tableau data model enum
+UPDATE dashboard_data_model_entity
+SET json = JSONB_SET(json::jsonb, '{dataModelType}', '"TableauDataModel"')
+WHERE json#>'{dataModelType}' = '"TableauSheet"';

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v012__create_db_connection_info.sql

@@ -0,0 +1,7 @@
+-- use FQN instead of name for Test Connection Definition
+ALTER TABLE test_connection_definition
+ADD fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
+DROP COLUMN name;
+
+-- Since we are not deleting the test connection defs anymore, clean it up
+TRUNCATE test_connection_definition;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v013__create_db_connection_info.sql

@@ -0,0 +1,193 @@
+-- we are not using the secretsManagerCredentials
+UPDATE metadata_service_entity
+SET json = json::jsonb #- '{openMetadataServerConnection.secretsManagerCredentials}'
+where name = 'OpenMetadata';
+
+-- Rename githubCredentials to gitCredentials
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,githubCredentials}', '{connection,config,gitCredentials}', json#>'{connection,config,githubCredentials}')
+    where serviceType = 'Looker'
+  and json#>'{connection,config,githubCredentials}' is not null;
+
+-- Rename gcsConfig in BigQuery to gcpConfig
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,credentials,gcsConfig}', '{connection,config,credentials,gcpConfig}',
+json#>'{connection,config,credentials,gcsConfig}')
+where serviceType in ('BigQuery')
+  and json#>'{connection,config,credentials,gcsConfig}' is not null;
+
+-- Rename gcsConfig in Datalake to gcpConfig
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,configSource,securityConfig,gcsConfig}', '{connection,config,configSource,securityConfig,gcpConfig}',
+json#>'{connection,config,configSource,securityConfig,gcsConfig}')
+where serviceType in ('Datalake')
+  and json#>'{connection,config,configSource,securityConfig,gcsConfig}' is not null;
+
+-- Rename gcsConfig in dbt to gcpConfig
+UPDATE ingestion_pipeline_entity
+SET json = jsonb_set(json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}', '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}', (json#>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}')::jsonb)
+WHERE json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig}' is not null and json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcsConfig}' is not null;
+
+
+-- Rename dashboardUrl in dashboard_entity to sourceUrl
+UPDATE dashboard_entity
+SET json = jsonb_set(json::jsonb #- '{dashboardUrl}' , '{sourceUrl}',
+json#>'{dashboardUrl}')
+where json#>'{dashboardUrl}' is not null;
+
+-- Rename chartUrl in chart_entity to sourceUr
+UPDATE chart_entity
+SET json = jsonb_set(json::jsonb #- '{chartUrl}' , '{sourceUrl}',
+json#>'{chartUrl}')
+where json#>'{chartUrl}' is not null;
+
+-- Rename pipelineUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity
+SET json = jsonb_set(json::jsonb #- '{pipelineUrl}' , '{sourceUrl}',
+json#>'{pipelineUrl}')
+where json#>'{pipelineUrl}' is not null;
+
+
+-- Rename taskUrl in pipeline_entity to sourceUrl
+UPDATE pipeline_entity
+SET json = jsonb_set(
+    json::jsonb - 'tasks',
+    '{tasks}',
+    (
+        SELECT jsonb_agg(
+            jsonb_build_object(
+                'name', t ->> 'name',
+                'sourceUrl', t ->> 'taskUrl',
+                'taskType', t ->> 'taskType',
+                'description', t ->> 'description',
+                'displayName', t ->> 'displayName',
+                'fullyQualifiedName', t ->> 'fullyQualifiedName',
+                'downstreamTasks', (t -> 'downstreamTasks')::jsonb,
+                'tags', (t ->> 'tags')::jsonb,
+                'endDate', t ->> 'endDate',
+                'startDate', t ->> 'startDate',
+                'taskSQL', t ->> 'taskSQL'
+            )
+        )
+        FROM jsonb_array_elements(json->'tasks') AS t
+    )
+)
+where json#>'{tasks}' is not null;
+
+
+-- Modify migrations for service connection of postgres and mysql to move password under authType
+UPDATE dbservice_entity
+SET json =  jsonb_set(
+json #-'{connection,config,password}',
+'{connection,config,authType}',
+jsonb_build_object('password',json#>'{connection,config,password}')
+)
+WHERE serviceType IN ('Postgres', 'Mysql')
+  and json#>'{connection,config,password}' is not null;
+
+-- Clean old test connections
+TRUNCATE automations_workflow;
+
+-- Remove sourceUrl in pipeline_entity from DatabricksPipeline & Fivetran
+UPDATE pipeline_entity
+SET json = json::jsonb #- '{sourceUrl}'
+where json #> '{serviceType}' in ('"DatabricksPipeline"','"Fivetran"');
+
+
+-- Remove sourceUrl in dashboard_entity from Mode
+UPDATE dashboard_entity
+SET json = json::jsonb #- '{sourceUrl}'
+where json #> '{serviceType}' in ('"Mode"');
+
+CREATE TABLE IF NOT EXISTS SERVER_CHANGE_LOG (
+    installed_rank SERIAL,
+    version VARCHAR(256) PRIMARY KEY,
+    migrationFileName VARCHAR(256) NOT NULL,
+    checksum VARCHAR(256) NOT NULL,
+    installed_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS SERVER_MIGRATION_SQL_LOGS (
+    version VARCHAR(256) NOT NULL,
+    sqlStatement VARCHAR(10000) NOT NULL,
+    checksum VARCHAR(256) PRIMARY KEY,
+    executedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Update test definition parameterValues
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMeanInCol", "dataType": "INT", "required": false, "description": "Expected mean value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueMeanToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForMedianInCol", "dataType": "INT", "required": false, "description": "Expected median value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueMedianToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be greater or equal than", "displayName": "Min", "optionValues": []}, {"name": "maxValueForStdDevInCol", "dataType": "INT", "required": false, "description": "Expected std. dev value for the column to be lower or equal than", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueStdDevToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minLength", "dataType": "INT", "required": false, "description": "The {minLength} for the column value. If minLength is not included, maxLength is treated as upperBound and there will be no minimum value length", "displayName": "Min", "optionValues": []}, {"name": "maxLength", "dataType": "INT", "required": false, "description": "The {maxLength} for the column value. if maxLength is not included, minLength is treated as lowerBound and there will be no maximum value length", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValueLengthsToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValue", "dataType": "INT", "required": false, "description": "The {minValue} value for the column entry. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "The {maxValue} value for the column entry. if maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'columnValuesToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "columnNames", "dataType": "STRING", "required": true, "description": "Expected columns names of the table to match the ones in {Column Names} -- should be a coma separated string", "displayName": "Column Names", "optionValues": []}, {"name": "ordered", "dataType": "BOOLEAN", "required": false, "description": "Whether or not to considered the order of the list when performing the match check", "displayName": "Ordered", "optionValues": []}]'
+        )
+where name = 'tableColumnToMatchSet';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name": "minValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be greater than or equal to {minValue}. If minValue is not included, maxValue is treated as upperBound and there will be no minimum", "displayName": "Min", "optionValues": []}, {"name": "maxValue", "dataType": "INT", "required": false, "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum", "displayName": "Max", "optionValues": []}]'
+        )
+where name = 'tableRowCountToBeBetween';
+
+update test_definition 
+set json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        '[{"name":"sqlExpression","displayName":"SQL Expression","description":"SQL expression to run against the table","dataType":"STRING","required":"true"},{"name":"strategy","displayName":"Strategy","description":"Strategy to use to run the custom SQL query (i.e. `SELECT COUNT(<col>)` or `SELECT <col> (defaults to ROWS)","dataType":"ARRAY","optionValues":["ROWS","COUNT"],"required":false},{"name":"threshold","displayName":"Threshold","description":"Threshold to use to determine if the test passes or fails (defaults to 0).","dataType":"NUMBER","required":false}]'
+        )
+where name = 'tableCustomSQLQuery';
+
+-- Modify migrations for service connection of airflow to move password under authType if 
+-- Connection Type as Mysql or Postgres
+
+UPDATE pipeline_service_entity
+SET json =  jsonb_set(
+json #-'{connection,config,connection,password}',
+'{connection,config,connection,authType}',
+jsonb_build_object('password',json#>'{connection,config,connection,password}')
+)
+WHERE serviceType = 'Airflow'
+and json#>'{connection,config,connection,type}' IN ('"Mysql"', '"Postgres"')
+and json#>'{connection,config,connection,password}' is not null;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v014__create_db_connection_info.sql

@@ -0,0 +1,43 @@
+-- Rename includeTempTables in snowflake to includeTransientTables
+
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb #- '{connection,config,includeTempTables}', '{connection,config,includeTransientTables}',
+json#>'{connection,config,includeTempTables}')
+where serviceType in ('Snowflake') and json#>'{connection,config,includeTempTables}' is not null ;
+
+
+update dbservice_entity
+set json = jsonb_set(json::jsonb, '{connection,config,scheme}', '"hive"')
+where json#>>'{connection,config,scheme}' in ('impala', 'impala4')
+and serviceType = 'Hive';
+
+-- remove the dataModel references from Data Models
+UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}';
+
+-- migrate ingestAllDatabases in mssql
+UPDATE dbservice_entity de2 
+SET json = JSONB_SET(
+    json || JSONB_SET(json,'{connection,config}', json#>'{connection,config}'|| 
+    jsonb_build_object('database',
+    (SELECT json->>'name' 
+        FROM database_entity de 
+        WHERE id = (SELECT er.toId 
+                    FROM entity_relationship er 
+                    WHERE er.fromId = de2.id 
+                    AND er.toEntity = 'database' 
+                    LIMIT 1)
+    )
+    )), 
+    '{connection,config,ingestAllDatabases}', 
+    'true'::jsonb
+)
+WHERE de2.serviceType = 'Mssql' 
+AND json->>'{connection,config,database}' IS NULL;
+
+-- remove keyfile from clickhouse
+UPDATE dbservice_entity
+SET json = json #-'{connection,config,keyfile}'
+WHERE serviceType = 'Clickhouse';
+
+-- Clean old test connections
+TRUNCATE automations_workflow;

bootstrap/sql/migrations/flyway/org.postgresql.Driver/v015__update_server_change_log.sql

@@ -0,0 +1,2 @@
+-- This column is already created in v000__create_server_change_log.sql
+-- ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics jsonb;