chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,114 @@
|
||||
-- matchEnum Test Definition Parameter for columnValuesToBeInSet
|
||||
UPDATE test_definition
|
||||
set json = JSON_MERGE_PRESERVE(
|
||||
json,
|
||||
'{"parameterDefinition": ['
|
||||
'{"name": "matchEnum", "displayName": "Match enum", "description": "If enabled, validate that each value independently matches the enum.", "dataType": "BOOLEAN", "required": false, "optionValues": []}'
|
||||
']}'
|
||||
)
|
||||
WHERE name = 'columnValuesToBeInSet'
|
||||
AND JSON_LENGTH(json, '$.parameterDefinition') < 2;
|
||||
|
||||
-- Test Case dyanic test migration
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(json, '$.supportsDynamicAssertion', true)
|
||||
WHERE name IN (
|
||||
'columnValueMaxToBeBetween',
|
||||
'columnValueMeanToBeBetween',
|
||||
'columnValueMedianToBeBetween',
|
||||
'columnValueMinToBeBetween',
|
||||
'columnValueStdDevToBeBetween',
|
||||
'columnValueLengthsToBeBetween',
|
||||
'columnValuesSumToBeBetween',
|
||||
'columnValuesToBeBetween',
|
||||
'tableRowCountToBeBetween'
|
||||
);
|
||||
|
||||
|
||||
-- KPI Migrations
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'percentage_of_data_asset_with_owner_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'PERCENTAGE')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'number_of_data_asset_with_owner_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'NUMBER')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'percentage_of_data_asset_with_description_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'PERCENTAGE')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'number_of_data_asset_with_description_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE JSON_EXTRACT(json, '$.metricType') = 'NUMBER')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
-- KPI MIgrations end
|
||||
|
||||
-- Update schedule type for applications
|
||||
UPDATE installed_apps
|
||||
SET json = JSON_MERGE_PATCH(json, '{"scheduleType": "ScheduledOrManual"}')
|
||||
WHERE JSON_UNQUOTE(json->'$.scheduleType') = 'Scheduled';
|
||||
|
||||
-- recreate all scheduled apps
|
||||
DELETE FROM apps_marketplace
|
||||
WHERE JSON_UNQUOTE(json->'$.scheduleType') = 'Scheduled';
|
||||
|
||||
ALTER table thread_entity DROP COLUMN entityId;
|
||||
|
||||
-- Add entityRef column to thread_entity table
|
||||
UPDATE thread_entity
|
||||
SET json = JSON_SET(
|
||||
JSON_REMOVE(
|
||||
JSON_REMOVE(json, '$.entityId'),
|
||||
'$.entityType'
|
||||
),
|
||||
'$.entityRef',
|
||||
JSON_OBJECT(
|
||||
'id', JSON_UNQUOTE(JSON_EXTRACT(json, '$.entityId')),
|
||||
'type', JSON_UNQUOTE(JSON_EXTRACT(json, '$.entityType'))
|
||||
)
|
||||
)
|
||||
WHERE JSON_CONTAINS_PATH(json, 'one', '$.entityId') OR JSON_CONTAINS_PATH(json, 'one', '$.entityType');
|
||||
|
||||
-- Add entityId and type column to thread_entity table
|
||||
ALTER table thread_entity ADD COLUMN entityId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityRef.id');
|
||||
ALTER table thread_entity ADD COLUMN entityType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.entityRef.type');
|
||||
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(json, '$.supportedDataTypes', JSON_ARRAY('NUMBER', 'TINYINT', 'SMALLINT', 'INT', 'BIGINT', 'BYTEINT', 'BYTES', 'FLOAT', 'DOUBLE', 'DECIMAL', 'NUMERIC', 'TIMESTAMP', 'TIMESTAMPZ', 'TIME', 'DATE', 'DATETIME', 'INTERVAL', 'STRING', 'MEDIUMTEXT', 'TEXT', 'CHAR', 'VARCHAR', 'BOOLEAN', 'BINARY', 'VARBINARY', 'BLOB', 'LONGBLOB', 'MEDIUMBLOB', 'MAP', 'STRUCT', 'UNION', 'SET', 'GEOGRAPHY', 'ENUM', 'UUID', 'VARIANT', 'GEOMETRY', 'POINT', 'POLYGON'))
|
||||
WHERE name = 'columnValuesToBeUnique';
|
||||
@@ -0,0 +1,336 @@
|
||||
-- Add a new table di_chart_entity
|
||||
CREATE TABLE IF NOT EXISTS di_chart_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
fqnHash varchar(768) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
UNIQUE(name),
|
||||
INDEX name_index (name)
|
||||
);
|
||||
|
||||
-- Update the KPI entity to remove the targetDefinition and set the targetValue to the value of the targetDefinition
|
||||
UPDATE kpi_entity
|
||||
SET json = JSON_REMOVE(
|
||||
JSON_SET(json,
|
||||
'$.targetValue',
|
||||
CAST(JSON_UNQUOTE(JSON_EXTRACT(json, '$.targetDefinition[0].value')) AS DECIMAL) * 100
|
||||
),
|
||||
'$.targetDefinition'
|
||||
)
|
||||
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.metricType')) = 'PERCENTAGE';
|
||||
|
||||
UPDATE kpi_entity
|
||||
SET json = JSON_REMOVE(
|
||||
JSON_SET(json,
|
||||
'$.targetValue',
|
||||
CAST(JSON_UNQUOTE(JSON_EXTRACT(json, '$.targetDefinition[0].value')) AS DECIMAL)
|
||||
),
|
||||
'$.targetDefinition'
|
||||
)
|
||||
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.metricType')) = 'NUMBER';
|
||||
-- Update DeltaLake service due to connection schema changes to enable DeltaLake ingestion from Storage
|
||||
UPDATE dbservice_entity dbse
|
||||
SET
|
||||
dbse.json = JSON_REMOVE(JSON_REMOVE(
|
||||
JSON_MERGE_PATCH(
|
||||
dbse.json,
|
||||
JSON_OBJECT(
|
||||
'connection', JSON_OBJECT(
|
||||
'config', JSON_OBJECT(
|
||||
'configSource', JSON_OBJECT(
|
||||
'connection', JSON_EXTRACT(dbse.json, '$.connection.config.metastoreConnection'),
|
||||
'appName', JSON_UNQUOTE(JSON_EXTRACT(dbse.json, '$.connection.config.appName'))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
, '$.connection.config.appName'), '$.connection.config.metastoreConnection')
|
||||
WHERE dbse.serviceType = 'DeltaLake';
|
||||
|
||||
-- Allow all bots to update the ingestion pipeline status
|
||||
UPDATE policy_entity
|
||||
SET json = JSON_ARRAY_APPEND(
|
||||
json,
|
||||
'$.rules',
|
||||
CAST('{
|
||||
"name": "BotRule-IngestionPipeline",
|
||||
"description": "A bot can Edit ingestion pipelines to pass the status",
|
||||
"resources": ["ingestionPipeline"],
|
||||
"operations": ["ViewAll","EditIngestionPipelineStatus"],
|
||||
"effect": "allow"
|
||||
}' AS JSON)
|
||||
)
|
||||
WHERE name = 'DefaultBotPolicy';
|
||||
|
||||
-- create API service entity
|
||||
CREATE TABLE IF NOT EXISTS api_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (nameHash),
|
||||
INDEX (name)
|
||||
);
|
||||
|
||||
-- create API collection entity
|
||||
CREATE TABLE IF NOT EXISTS api_collection_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(256) NOT NULL COLLATE ascii_bin,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash),
|
||||
INDEX (name)
|
||||
);
|
||||
|
||||
-- create API Endpoint entity
|
||||
CREATE TABLE IF NOT EXISTS api_endpoint_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(256) NOT NULL COLLATE ascii_bin,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash),
|
||||
INDEX (name)
|
||||
);
|
||||
|
||||
-- Clean dangling workflows not removed after test connection
|
||||
truncate automations_workflow;
|
||||
|
||||
-- Remove date, dateTime, time from type_entity, as they are no more om-field-types, instead we have date-cp, time-cp, dateTime-cp as om-field-types
|
||||
DELETE FROM type_entity
|
||||
WHERE name IN ('date', 'dateTime', 'time');
|
||||
|
||||
-- Update BigQuery,Bigtable & Datalake model for gcpCredentials to move `gcpConfig` value to `gcpConfig.path`
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.credentials.gcpConfig'),
|
||||
'$.connection.config.credentials.gcpConfig',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.credentials.gcpConfig.path',
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig')
|
||||
) where serviceType in ('BigQuery', 'BigTable') and
|
||||
(JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.type') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.externalType') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.path')) is NULL;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.configSource.securityConfig.gcpConfig'),
|
||||
'$.connection.config.configSource.securityConfig.gcpConfig',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.configSource.securityConfig.gcpConfig.path',
|
||||
JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcpConfig')
|
||||
) where serviceType in ('Datalake') and
|
||||
(JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcpConfig.type') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcpConfig.externalType') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcpConfig.path')) is NULL and
|
||||
JSON_EXTRACT(json, '$.connection.config.configSource.securityConfig.gcpConfig') is NOT NULL;
|
||||
|
||||
-- Update Powerbi model for pbitFilesSource to move `gcpConfig` value to `gcpConfig.path`
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig'),
|
||||
'$.connection.config.pbitFilesSource.securityConfig.gcpConfig',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.pbitFilesSource.securityConfig.gcpConfig.path',
|
||||
JSON_EXTRACT(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig')
|
||||
) where serviceType in ('PowerBI') and
|
||||
(JSON_EXTRACT(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig.type') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig.externalType') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig.path')) is NULL AND
|
||||
JSON_EXTRACT(json, '$.connection.config.pbitFilesSource.securityConfig.gcpConfig') is not null;
|
||||
|
||||
UPDATE storage_service_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.credentials.gcpConfig'),
|
||||
'$.connection.config.credentials.gcpConfig',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.credentials.gcpConfig.path',
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig')
|
||||
) where serviceType in ('GCS') and
|
||||
(JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.type') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.externalType') OR
|
||||
JSON_EXTRACT(json, '$.connection.config.credentials.gcpConfig.path')) is NULL;
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig'),
|
||||
'$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig',
|
||||
JSON_OBJECT(),
|
||||
'$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.path',
|
||||
JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig')
|
||||
) where JSON_EXTRACT(json, '$.sourceConfig.config.type') = 'DBT' and (
|
||||
JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.type') OR
|
||||
JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.externalType') OR
|
||||
JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig.path')
|
||||
) is NULL AND JSON_EXTRACT(json, '$.sourceConfig.config.dbtConfigSource.dbtSecurityConfig.gcpConfig') is not null;
|
||||
|
||||
-- Update Owner Field to Owners
|
||||
DELETE from event_subscription_entity where name = 'ActivityFeedAlert';
|
||||
|
||||
-- Update thread_entity to move previousOwner and updatedOwner to array
|
||||
UPDATE thread_entity
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.feedInfo.entitySpecificInfo.previousOwner',
|
||||
JSON_ARRAY(
|
||||
JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.previousOwner')
|
||||
)
|
||||
)
|
||||
WHERE JSON_CONTAINS_PATH(json, 'one', '$.feedInfo.entitySpecificInfo.previousOwner')
|
||||
AND JSON_TYPE(JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.previousOwner')) <> 'ARRAY';
|
||||
|
||||
UPDATE thread_entity
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.feedInfo.entitySpecificInfo.updatedOwner',
|
||||
JSON_ARRAY(
|
||||
JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.updatedOwner')
|
||||
)
|
||||
)
|
||||
WHERE JSON_CONTAINS_PATH(json, 'one', '$.feedInfo.entitySpecificInfo.updatedOwner')
|
||||
AND JSON_TYPE(JSON_EXTRACT(json, '$.feedInfo.entitySpecificInfo.updatedOwner')) <> 'ARRAY';
|
||||
|
||||
-- Update entity_extension to move owner to array
|
||||
update entity_extension set json = JSON_SET(
|
||||
JSON_REMOVE(json, '$.owner'),
|
||||
'$.owners',
|
||||
JSON_ARRAY(
|
||||
JSON_EXTRACT(json, '$.owner')
|
||||
)
|
||||
) where json -> '$.owner' is not null;
|
||||
|
||||
ALTER TABLE test_case MODIFY COLUMN `name` VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
|
||||
-- set templates to fetch emailTemplates
|
||||
UPDATE openmetadata_settings
|
||||
SET json = JSON_SET(json, '$.templates', 'openmetadata')
|
||||
WHERE configType = 'emailConfiguration';
|
||||
|
||||
-- remove dangling owner and service from ingestion pipelines. This info is in entity_relationship
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json, '$.owner', '$.service');
|
||||
|
||||
ALTER TABLE thread_entity ADD COLUMN domain VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.domain');
|
||||
|
||||
-- Remove owner from json from all entities
|
||||
update api_collection_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update api_endpoint_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update api_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update bot_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update chart_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update dashboard_data_model_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update dashboard_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update dashboard_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update data_product_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update database_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update database_schema_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update dbservice_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update di_chart_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update domain_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update event_subscription_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update glossary_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update glossary_term_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update ingestion_pipeline_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update kpi_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update messaging_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update metadata_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update metric_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update ml_model_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update mlmodel_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update persona_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update pipeline_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update pipeline_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update policy_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update query_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update report_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update role_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update search_index_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update search_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update storage_container_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update storage_service_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update stored_procedure_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update table_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update team_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update thread_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update topic_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update type_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update user_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update test_case set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update installed_apps set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update apps_marketplace set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update classification set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update storage_container_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update data_insight_chart set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update doc_store set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update tag set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update test_connection_definition set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update test_definition set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update test_suite set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update topic_entity set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update web_analytic_event set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
update automations_workflow set json = JSON_REMOVE(json, '$.owner') where json -> '$.owner' is not null;
|
||||
|
||||
update table_entity set json = JSON_SET(
|
||||
JSON_REMOVE(json, '$.dataModel.owner'),
|
||||
'$.dataModel.owners',
|
||||
JSON_ARRAY(
|
||||
JSON_EXTRACT(json, '$.dataModel.owner')
|
||||
)
|
||||
) where json -> '$.dataModel.owner' is not null;
|
||||
|
||||
|
||||
ALTER TABLE automations_workflow DROP COLUMN status, DROP COLUMN workflowType;
|
||||
ALTER TABLE automations_workflow
|
||||
ADD COLUMN status VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.status') STORED,
|
||||
ADD COLUMN workflowType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.workflowType') STORED NOT NULL;
|
||||
|
||||
ALTER TABLE entity_extension ADD INDEX extension_index(extension);
|
||||
|
||||
ALTER TABLE test_definition MODIFY COLUMN `name` VARCHAR(512) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
|
||||
-- Remove SearchIndexing for api Service, collection and endpoint
|
||||
DELETE er FROM entity_relationship er JOIN installed_apps ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
|
||||
DELETE er FROM entity_relationship er JOIN apps_marketplace ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
|
||||
DELETE from installed_apps where name = 'SearchIndexingApplication';
|
||||
DELETE from apps_marketplace where name = 'SearchIndexingApplication';
|
||||
|
||||
-- Drop the existing taskAssigneesIds
|
||||
DROP INDEX taskAssigneesIds_index ON thread_entity;
|
||||
|
||||
ALTER TABLE thread_entity DROP COLUMN taskAssigneesIds;
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (
|
||||
REPLACE(
|
||||
REPLACE(
|
||||
JSON_UNQUOTE(
|
||||
JSON_EXTRACT(taskAssignees, '$[*].id')
|
||||
), '[', ''
|
||||
), ']', ''
|
||||
)
|
||||
) STORED;
|
||||
|
||||
CREATE FULLTEXT INDEX taskAssigneesIds_index ON thread_entity(taskAssigneesIds);
|
||||
|
||||
-- Add indexes on thread_entity and entity_relationship to improve count/feed api performance
|
||||
CREATE INDEX idx_thread_entity_entityId_createdAt ON thread_entity (entityId, createdAt);
|
||||
CREATE INDEX idx_thread_entity_id_type_status ON thread_entity (id, type, taskStatus);
|
||||
CREATE INDEX idx_er_fromEntity_fromId_toEntity_relation ON entity_relationship (fromEntity, fromId, toEntity, relation);
|
||||
@@ -0,0 +1,114 @@
|
||||
-- matchEnum Test Definition Parameter for columnValuesToBeInSet
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(json, '{parameterDefinition}', json->'parameterDefinition' || '['
|
||||
'{"name": "matchEnum", "displayName": "Match enum", "description": "If enabled, validate that each value independently matches the enum.", "dataType": "BOOLEAN", "required": false, "optionValues": []}'
|
||||
']'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeInSet'
|
||||
AND JSONB_ARRAY_LENGTH(json->'parameterDefinition') < 2;
|
||||
|
||||
|
||||
-- Test Case dyanic test migration
|
||||
UPDATE test_definition
|
||||
SET json = JSONB_SET(json, '{supportsDynamicAssertion}', 'true', true)
|
||||
WHERE name IN (
|
||||
'columnValueMaxToBeBetween',
|
||||
'columnValueMeanToBeBetween',
|
||||
'columnValueMedianToBeBetween',
|
||||
'columnValueMinToBeBetween',
|
||||
'columnValueStdDevToBeBetween',
|
||||
'columnValueLengthsToBeBetween',
|
||||
'columnValuesSumToBeBetween',
|
||||
'columnValuesToBeBetween',
|
||||
'tableRowCountToBeBetween'
|
||||
);
|
||||
|
||||
|
||||
-- KPI Migrations
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'percentage_of_data_asset_with_owner_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'PERCENTAGE')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'number_of_data_asset_with_owner_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithOwnerByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'NUMBER')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'percentage_of_data_asset_with_description_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'PERCENTAGE')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
|
||||
|
||||
UPDATE entity_relationship
|
||||
SET toid = (SELECT id
|
||||
FROM di_chart_entity
|
||||
WHERE NAME = 'number_of_data_asset_with_description_kpi'),
|
||||
toentity = 'dataInsightCustomChart'
|
||||
WHERE toid = (SELECT id
|
||||
FROM data_insight_chart dic
|
||||
WHERE NAME = 'PercentageOfEntitiesWithDescriptionByType')
|
||||
AND fromId IN (SELECT id from kpi_entity WHERE json ->> 'metricType' = 'NUMBER')
|
||||
AND toentity = 'dataInsightChart'
|
||||
AND fromentity = 'kpi';
|
||||
-- KPI MIgrations end
|
||||
|
||||
-- Update schedule type for applications
|
||||
UPDATE installed_apps
|
||||
SET json = json || '{"scheduleType": "ScheduledOrManual"}'
|
||||
WHERE json->>'scheduleType' = 'Scheduled';
|
||||
|
||||
-- recreate all scheduled apps
|
||||
DELETE FROM apps_marketplace
|
||||
WHERE json->>'scheduleType' = 'Scheduled';
|
||||
|
||||
ALTER TABLE thread_entity DROP COLUMN entityId;
|
||||
|
||||
-- Add entityRef column to thread_entity table
|
||||
UPDATE thread_entity
|
||||
SET json = jsonb_set(
|
||||
json - 'entityId' - 'entityType',
|
||||
'{entityRef}',
|
||||
jsonb_build_object(
|
||||
'id', json->>'entityId',
|
||||
'type', json->>'entityType'
|
||||
),
|
||||
true
|
||||
)
|
||||
WHERE jsonb_exists(json, 'entityId') OR jsonb_exists(json, 'entityType');
|
||||
|
||||
-- Add entityId and type column to thread_entity table
|
||||
ALTER TABLE thread_entity ADD COLUMN entityId VARCHAR(36) GENERATED ALWAYS AS (json->'entityRef'->>'id') STORED;
|
||||
ALTER TABLE thread_entity ADD COLUMN entityType VARCHAR(36) GENERATED ALWAYS AS (json->'entityRef'->>'type') STORED;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{supportedDataTypes}',
|
||||
'["NUMBER", "TINYINT", "SMALLINT", "INT", "BIGINT", "BYTEINT", "BYTES", "FLOAT", "DOUBLE", "DECIMAL", "NUMERIC", "TIMESTAMP", "TIMESTAMPZ", "TIME", "DATE", "DATETIME", "INTERVAL", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN", "BINARY", "VARBINARY", "BLOB", "LONGBLOB", "MEDIUMBLOB", "MAP", "STRUCT", "UNION", "SET", "GEOGRAPHY", "ENUM", "UUID", "VARIANT", "GEOMETRY", "POINT", "POLYGON"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeUnique';
|
||||
@@ -0,0 +1,307 @@
|
||||
-- Add a new table di_chart_entity
|
||||
CREATE TABLE IF NOT EXISTS di_chart_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) DEFAULT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::bool) STORED,
|
||||
UNIQUE(name)
|
||||
);
|
||||
|
||||
UPDATE kpi_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{targetValue}',
|
||||
to_jsonb((json->'targetDefinition'->0->>'value')::numeric * 100)
|
||||
) #- '{targetDefinition}'
|
||||
WHERE json->>'metricType' = 'PERCENTAGE';
|
||||
|
||||
UPDATE kpi_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{targetValue}',
|
||||
to_jsonb((json->'targetDefinition'->0->>'value')::numeric)
|
||||
) #- '{targetDefinition}'
|
||||
WHERE json->>'metricType' = 'NUMBER';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSONB_SET(
|
||||
JSONB_SET(
|
||||
json,
|
||||
'{connection,config,configSource}',
|
||||
JSONB_BUILD_OBJECT('connection', json->'connection'->'config'->'metastoreConnection')
|
||||
),
|
||||
'{connection,config,configSource,appName}',
|
||||
json->'connection'->'config'->'appName'
|
||||
) #- '{connection,config,metastoreConnection}' #- '{connection,config,appName}'
|
||||
WHERE serviceType = 'DeltaLake';
|
||||
|
||||
|
||||
-- Allow all bots to update the ingestion pipeline status
|
||||
UPDATE policy_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{rules}',
|
||||
(json->'rules')::jsonb || to_jsonb(ARRAY[
|
||||
jsonb_build_object(
|
||||
'name', 'BotRule-IngestionPipeline',
|
||||
'description', 'A bot can Edit ingestion pipelines to pass the status',
|
||||
'resources', jsonb_build_array('ingestionPipeline'),
|
||||
'operations', jsonb_build_array('ViewAll', 'EditIngestionPipelineStatus'),
|
||||
'effect', 'allow'
|
||||
)
|
||||
]),
|
||||
true
|
||||
)
|
||||
WHERE json->>'name' = 'DefaultBotPolicy';
|
||||
|
||||
-- create API service entity
|
||||
CREATE TABLE IF NOT EXISTS api_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
nameHash VARCHAR(256) NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (nameHash)
|
||||
);
|
||||
|
||||
-- create API collection entity
|
||||
CREATE TABLE IF NOT EXISTS api_collection_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(256) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
-- create API Endpoint entity
|
||||
CREATE TABLE IF NOT EXISTS api_endpoint_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(256) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
-- Clean dangling workflows not removed after test connection
|
||||
truncate automations_workflow;
|
||||
|
||||
-- Remove date, dateTime, time from type_entity, as they are no more om-field-types, instead we have date-cp, time-cp, dateTime-cp as om-field-types
|
||||
DELETE FROM type_entity
|
||||
WHERE name IN ('date', 'dateTime', 'time');
|
||||
|
||||
-- Update BigQuery,Bigtable & Datalake model for gcpCredentials to move `gcpConfig` value to `gcpConfig.path`
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,credentials,gcpConfig}',
|
||||
'{connection,config,credentials,gcpConfig}',
|
||||
jsonb_build_object('path', json#>'{connection,config,credentials,gcpConfig}')
|
||||
)
|
||||
WHERE serviceType IN ('BigQuery', 'BigTable') and json#>>'{connection,config,credentials,gcpConfig}' is not null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,type}' is null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,externalType}' is null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,path}' is null;
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,configSource,securityConfig,gcpConfig}',
|
||||
'{connection,config,configSource,securityConfig,gcpConfig}',
|
||||
jsonb_build_object('path', json#>'{connection,config,configSource,securityConfig,gcpConfig}')
|
||||
)
|
||||
WHERE serviceType IN ('Datalake') and json#>>'{connection,config,configSource,securityConfig,gcpConfig}' is not null
|
||||
and json#>>'{connection,config,configSource,securityConfig,gcpConfig,type}' is null
|
||||
and json#>>'{connection,config,configSource,securityConfig,gcpConfig,externalType}' is null
|
||||
and json#>>'{connection,config,configSource,securityConfig,gcpConfig,path}' is null;
|
||||
|
||||
|
||||
-- Update Powerbi model for pbitFilesSource to move `gcpConfig` value to `gcpConfig.path`
|
||||
|
||||
UPDATE dashboard_service_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,pbitFilesSource,securityConfig,gcpConfig}',
|
||||
'{connection,config,pbitFilesSource,securityConfig,gcpConfig}',
|
||||
jsonb_build_object('path', json#>'{connection,config,pbitFilesSource,securityConfig,gcpConfig}')
|
||||
)
|
||||
WHERE serviceType IN ('PowerBI') and
|
||||
json#>>'{connection,config,pbitFilesSource,securityConfig,gcpConfig}' is not null
|
||||
and json#>>'{connection,config,pbitFilesSource,securityConfig,gcpConfig,type}' is null
|
||||
and json#>>'{connection,config,pbitFilesSource,securityConfig,gcpConfig,externalType}' is null
|
||||
and json#>>'{connection,config,pbitFilesSource,securityConfig,gcpConfig,path}' is null;
|
||||
|
||||
UPDATE storage_service_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,credentials,gcpConfig}',
|
||||
'{connection,config,credentials,gcpConfig}',
|
||||
jsonb_build_object('path', json#>'{connection,config,credentials,gcpConfig}')
|
||||
) where serviceType = 'GCS' and
|
||||
json#>>'{connection,config,credentials,gcpConfig}' is not null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,type}' is null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,externalType}' is null
|
||||
and json#>>'{connection,config,credentials,gcpConfig,path}' is null;
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = jsonb_set(
|
||||
json::jsonb #- '{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}'::text[],
|
||||
'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}',
|
||||
jsonb_build_object('path', json#>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}')
|
||||
)
|
||||
WHERE json#>>'{sourceConfig,config,type}' = 'DBT'
|
||||
AND json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig}' IS NOT NULL
|
||||
AND json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig,type}' IS NULL
|
||||
AND json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig,externalType}' IS NULL
|
||||
AND json#>>'{sourceConfig,config,dbtConfigSource,dbtSecurityConfig,gcpConfig,path}' IS NULL;
|
||||
|
||||
-- Update Owner Field to Owners
|
||||
DELETE from event_subscription_entity where name = 'ActivityFeedAlert';
|
||||
|
||||
-- Update thread_entity to move previousOwner and updatedOwner to array
|
||||
UPDATE thread_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{feedInfo,entitySpecificInfo,previousOwner}',
|
||||
to_jsonb(ARRAY[json->'feedInfo'->'entitySpecificInfo'->'previousOwner'])
|
||||
)
|
||||
WHERE jsonb_path_exists(json, '$.feedInfo.entitySpecificInfo.previousOwner')
|
||||
AND jsonb_path_query_first(json, '$.feedInfo.entitySpecificInfo.previousOwner ? (@ != null)') IS NOT null
|
||||
AND jsonb_typeof(json->'feedInfo'->'entitySpecificInfo'->'updatedOwner') <> 'array';
|
||||
|
||||
UPDATE thread_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{feedInfo,entitySpecificInfo,updatedOwner}',
|
||||
to_jsonb(ARRAY[json->'feedInfo'->'entitySpecificInfo'->'updatedOwner'])
|
||||
)
|
||||
WHERE jsonb_path_exists(json, '$.feedInfo.entitySpecificInfo.updatedOwner')
|
||||
AND jsonb_path_query_first(json, '$.feedInfo.entitySpecificInfo.updatedOwner ? (@ != null)') IS NOT null
|
||||
AND jsonb_typeof(json->'feedInfo'->'entitySpecificInfo'->'updatedOwner') <> 'array';
|
||||
|
||||
-- Update entity_extension to move owner to array
|
||||
update entity_extension set json = jsonb_set(json#-'{owner}', '{owners}',
|
||||
jsonb_build_array(json#>'{owner}')) where json #>> '{owner}' is not null;
|
||||
|
||||
ALTER TABLE test_case ALTER COLUMN name TYPE VARCHAR(512);
|
||||
|
||||
-- set templates to fetch emailTemplates
|
||||
UPDATE openmetadata_settings
|
||||
SET json = jsonb_set(json, '{templates}', '"openmetadata"')
|
||||
WHERE configType = 'emailConfiguration';
|
||||
|
||||
-- remove dangling owner and service from ingestion pipelines. This info is in entity_relationship
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{owner}'
|
||||
WHERE json #> '{owner}' IS NOT NULL;
|
||||
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{service}'
|
||||
WHERE json #> '{service}' IS NOT NULL;
|
||||
|
||||
ALTER TABLE thread_entity ADD COLUMN domain VARCHAR(256) GENERATED ALWAYS AS (json ->> 'domain') STORED;
|
||||
|
||||
-- Remove owner from json from all entities
|
||||
|
||||
update api_collection_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update api_endpoint_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update api_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update bot_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update chart_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update dashboard_data_model_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update dashboard_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update dashboard_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update data_product_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update database_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update database_schema_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update dbservice_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update di_chart_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update domain_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update event_subscription_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update glossary_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update glossary_term_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update ingestion_pipeline_entity set json = json::jsonb#-'{owner}' where json::jsonb #>> '{owner}' is not null;
|
||||
update kpi_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update messaging_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update metadata_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update metric_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update ml_model_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update mlmodel_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update persona_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update pipeline_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update pipeline_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update policy_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update query_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update report_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update role_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update search_index_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update search_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update storage_container_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update storage_service_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update stored_procedure_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update table_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update team_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update thread_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update topic_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update type_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update user_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update test_case set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update installed_apps set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update apps_marketplace set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update classification set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update storage_container_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update data_insight_chart set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update doc_store set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update tag set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update test_connection_definition set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update test_definition set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update test_suite set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update topic_entity set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update web_analytic_event set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
update automations_workflow set json = json#-'{owner}' where json #>> '{owner}' is not null;
|
||||
|
||||
update table_entity set json = jsonb_set(json#-'{dataModel,owner}', '{dataModel,owners}',
|
||||
jsonb_build_array(json#>'{dataModel,owner}')) where json #>> '{dataModel,owner}' is not null;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS extension_index ON entity_extension (extension);
|
||||
|
||||
ALTER TABLE test_definition ALTER COLUMN name TYPE VARCHAR(512);
|
||||
|
||||
-- Remove SearchIndexing for api Service, collection and endpoint
|
||||
DELETE FROM entity_relationship er USING installed_apps ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication';
|
||||
DELETE FROM entity_relationship er USING apps_marketplace ia WHERE (er.fromId = ia.id OR er.toId = ia.id) AND ia.name = 'SearchIndexingApplication';
|
||||
DELETE from installed_apps where name = 'SearchIndexingApplication';
|
||||
DELETE from apps_marketplace where name = 'SearchIndexingApplication';
|
||||
|
||||
-- Drop the existing taskAssigneesIds
|
||||
DROP INDEX IF EXISTS taskAssigneesIds_index;
|
||||
|
||||
ALTER TABLE thread_entity DROP COLUMN IF EXISTS taskAssigneesIds;
|
||||
|
||||
ALTER TABLE thread_entity
|
||||
ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (
|
||||
TRIM(BOTH '[]' FROM (
|
||||
(jsonb_path_query_array(json, '$.task.assignees[*].id'))::TEXT
|
||||
))
|
||||
) STORED;
|
||||
|
||||
|
||||
CREATE INDEX idx_task_assignees_ids_fulltext
|
||||
ON thread_entity USING GIN (to_tsvector('simple', taskAssigneesIds));
|
||||
|
||||
|
||||
-- Add indexes on thread_entity and entity_relationship to improve count/feed api performance
|
||||
CREATE INDEX idx_thread_entity_entityId_createdAt ON thread_entity (entityId, createdAt);
|
||||
CREATE INDEX idx_thread_entity_id_type_status ON thread_entity (id, type, taskStatus);
|
||||
CREATE INDEX idx_er_fromEntity_fromId_toEntity_relation ON entity_relationship (fromEntity, fromId, toEntity, relation);
|
||||
Reference in New Issue
Block a user