chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,43 @@
|
||||
ALTER TABLE bot_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE chart_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE classification ADD UNIQUE (nameHash);
|
||||
ALTER TABLE storage_container_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE dashboard_data_model_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE dashboard_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE dashboard_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE data_insight_chart ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE database_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE database_schema_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE dbservice_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE event_subscription_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE glossary_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE glossary_term_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE ingestion_pipeline_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE kpi_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE messaging_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE metadata_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE metric_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE ml_model_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE mlmodel_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE pipeline_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE pipeline_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE policy_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE query_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE report_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE role_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE storage_service_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE table_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE tag ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE team_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE test_case ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE test_connection_definition ADD UNIQUE (nameHash);
|
||||
ALTER TABLE test_definition ADD UNIQUE (nameHash);
|
||||
ALTER TABLE test_suite ADD UNIQUE (nameHash);
|
||||
ALTER TABLE topic_entity ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE type_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE user_entity ADD UNIQUE (nameHash);
|
||||
ALTER TABLE web_analytic_event ADD UNIQUE (fqnHash);
|
||||
ALTER TABLE automations_workflow ADD UNIQUE (nameHash);
|
||||
ALTER TABLE entity_extension_time_series DROP COLUMN entityFQN;
|
||||
ALTER TABLE field_relationship DROP KEY `PRIMARY`,ADD CONSTRAINT `field_relationship_primary` PRIMARY KEY(fromFQNHash, toFQNHash, relation), MODIFY fromFQN VARCHAR(2096) NOT NULL, MODIFY toFQN VARCHAR(2096) NOT NULL;
|
||||
ALTER TABLE tag_usage DROP index `source`, DROP COLUMN targetFQN, ADD UNIQUE KEY `tag_usage_key` (source, tagFQNHash, targetFQNHash);
|
||||
@@ -0,0 +1,44 @@
|
||||
UPDATE query_entity SET json = JSON_INSERT(json, '$.fullyQualifiedName', JSON_EXTRACT(json, '$.name')) WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') IS NULL;
|
||||
ALTER TABLE bot_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE chart_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE classification DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE storage_container_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE dashboard_data_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE dashboard_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE dashboard_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE data_insight_chart DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
|
||||
ALTER TABLE database_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE database_schema_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE dbservice_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE event_subscription_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE glossary_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE glossary_term_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE ingestion_pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE kpi_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE messaging_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE metadata_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE metric_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE ml_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE mlmodel_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE pipeline_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE policy_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE query_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE report_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE role_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE storage_service_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE table_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE tag DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE team_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE test_case DROP COLUMN fullyQualifiedName, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
|
||||
ALTER TABLE test_connection_definition ADD COLUMN nameHash VARCHAR(256) NOT NULL,ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE test_definition DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE test_suite DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE topic_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL;
|
||||
ALTER TABLE type_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE user_entity DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE web_analytic_event DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768) NOT NULL;
|
||||
ALTER TABLE automations_workflow DROP KEY `name`, ADD COLUMN nameHash VARCHAR(256) NOT NULL;
|
||||
ALTER TABLE field_relationship ADD COLUMN fromFQNHash VARCHAR(382), ADD COLUMN toFQNHash VARCHAR(382), DROP INDEX from_index, DROP INDEX to_index, ADD INDEX from_fqnhash_index(fromFQNHash, relation), ADD INDEX to_fqnhash_index(toFQNHash, relation);
|
||||
ALTER TABLE entity_extension_time_series ADD COLUMN entityFQNHash VARCHAR (768) NOT NULL;
|
||||
ALTER TABLE tag_usage ADD COLUMN tagFQNHash VARCHAR(382), ADD COLUMN targetFQNHash VARCHAR(382);
|
||||
@@ -0,0 +1,43 @@
|
||||
ALTER TABLE bot_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE chart_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE classification ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE storage_container_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE dashboard_data_model_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE dashboard_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE dashboard_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE data_insight_chart ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE database_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE database_schema_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE dbservice_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE event_subscription_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE glossary_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE glossary_term_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE ingestion_pipeline_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE kpi_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE messaging_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE metadata_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE metric_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE ml_model_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE mlmodel_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE pipeline_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE pipeline_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE policy_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE query_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE report_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE role_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE storage_service_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE table_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE tag ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE team_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE test_case ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE test_connection_definition ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE test_definition ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE test_suite ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE topic_entity ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE type_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE user_entity ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE web_analytic_event ADD UNIQUE (fqnHash), ALTER COLUMN fqnHash SET NOT NULL;
|
||||
ALTER TABLE automations_workflow ADD UNIQUE (nameHash), ALTER COLUMN nameHash SET NOT NULL;
|
||||
ALTER TABLE field_relationship DROP CONSTRAINT field_relationship_pkey, ADD CONSTRAINT field_relationship_pkey PRIMARY KEY(fromFQNHash, toFQNHash, relation), ALTER fromFQN TYPE VARCHAR(2096), ALTER toFQN TYPE VARCHAR(2096);
|
||||
ALTER TABLE entity_extension_time_series DROP COLUMN entityFQN, ALTER COLUMN entityFQNHash SET NOT NULL;
|
||||
ALTER TABLE tag_usage DROP CONSTRAINT tag_usage_source_tagfqn_targetfqn_key, DROP COLUMN targetFQN, ADD UNIQUE (source, tagFQNHash, targetFQNHash);
|
||||
@@ -0,0 +1,47 @@
|
||||
UPDATE query_entity SET json = jsonb_set(json::jsonb, '{fullyQualifiedName}', json#>'{name}') WHERE json#>'{fullyQualifiedName}' IS NULL;
|
||||
ALTER TABLE bot_entity DROP CONSTRAINT bot_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE chart_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE classification DROP CONSTRAINT tag_category_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE storage_container_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE dashboard_data_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE dashboard_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE dashboard_service_entity DROP CONSTRAINT dashboard_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE data_insight_chart DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768);
|
||||
ALTER TABLE database_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE database_schema_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE dbservice_entity DROP CONSTRAINT dbservice_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE event_subscription_entity DROP CONSTRAINT event_subscription_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE glossary_entity DROP CONSTRAINT glossary_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE glossary_term_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE ingestion_pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE kpi_entity DROP CONSTRAINT kpi_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE messaging_service_entity DROP CONSTRAINT messaging_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE metadata_service_entity DROP CONSTRAINT metadata_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE metric_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE ml_model_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE mlmodel_service_entity DROP CONSTRAINT mlmodel_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE pipeline_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE pipeline_service_entity DROP CONSTRAINT pipeline_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE policy_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE query_entity DROP CONSTRAINT query_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE report_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE role_entity DROP CONSTRAINT role_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE storage_service_entity DROP CONSTRAINT storage_service_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE table_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE tag DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE team_entity DROP CONSTRAINT team_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE test_case DROP COLUMN fullyQualifiedName, ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, ADD COLUMN fqnHash VARCHAR(768);
|
||||
ALTER TABLE test_connection_definition ADD COLUMN nameHash VARCHAR(256), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE test_definition DROP CONSTRAINT test_definition_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE test_suite DROP CONSTRAINT test_suite_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE topic_entity DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(256), ADD COLUMN name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL;
|
||||
ALTER TABLE type_entity DROP CONSTRAINT type_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE user_entity DROP CONSTRAINT user_entity_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
ALTER TABLE web_analytic_event DROP COLUMN fullyQualifiedName, ADD COLUMN fqnHash VARCHAR(768);
|
||||
ALTER TABLE automations_workflow DROP CONSTRAINT automations_workflow_name_key, ADD COLUMN nameHash VARCHAR(256);
|
||||
DROP INDEX field_relationship_from_index, field_relationship_to_index;
|
||||
ALTER TABLE field_relationship ADD COLUMN fromFQNHash VARCHAR(382), ADD COLUMN toFQNHash VARCHAR(382);
|
||||
CREATE INDEX IF NOT EXISTS field_relationship_from_index ON field_relationship(fromFQNHash, relation);
|
||||
CREATE INDEX IF NOT EXISTS field_relationship_to_index ON field_relationship(toFQNHash, relation);
|
||||
ALTER TABLE entity_extension_time_series ADD COLUMN entityFQNHash VARCHAR (768);
|
||||
ALTER TABLE tag_usage ADD COLUMN tagFQNHash VARCHAR(382), ADD COLUMN targetFQNHash VARCHAR(382);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE test_suite CHANGE COLUMN nameHash fqnHash VARCHAR(256);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE test_suite RENAME COLUMN nameHash TO fqnHash;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- queries to rename params to connectionOptions for trino
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_SET(
|
||||
JSON_REMOVE(json, '$.connection.config.connectionOptions'),
|
||||
'$.connection.config.connectionOptions',
|
||||
JSON_EXTRACT(json, '$.connection.config.params')
|
||||
)
|
||||
WHERE serviceType = 'Trino';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_REMOVE(json, '$.connection.config.params')
|
||||
WHERE serviceType = 'Trino';
|
||||
|
||||
-- Modify migrations for service connection of trino to move password under authType
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.password'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT(),
|
||||
'$.connection.config.authType.password',
|
||||
JSON_EXTRACT(json, '$.connection.config.password'))
|
||||
where serviceType = 'Trino'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- queries to rename params to connectionOptions for trino
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{connection,config,connectionOptions}',
|
||||
jsonb_extract_path(json, 'connection', 'config', 'params'),
|
||||
true
|
||||
)
|
||||
WHERE serviceType = 'Trino';
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = json::jsonb #- '{connection,config,params}'
|
||||
where json #> '{serviceType}' in ('"Trino"');
|
||||
|
||||
-- Modify migrations for service connection of trino to move password under authType
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #-'{connection,config,password}',
|
||||
'{connection,config,authType}',
|
||||
jsonb_build_object('password',json#>'{connection,config,password}')
|
||||
)
|
||||
WHERE serviceType = 'Trino'
|
||||
and json#>'{connection,config,password}' is not null;
|
||||
@@ -0,0 +1,50 @@
|
||||
START TRANSACTION;
|
||||
-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
|
||||
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> '$.id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> '$.reportDataType' = 'WebAnalyticEntityViewReportData'
|
||||
) duplicates
|
||||
ORDER BY `date` DESC, `timestamp` DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
|
||||
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> '$.id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> '$.reportDataType' = 'EntityReportData'
|
||||
) duplicates
|
||||
ORDER BY `date` DESC, `timestamp` DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
|
||||
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> '$.id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> '$.reportDataType' = 'WebAnalyticUserActivityReportData'
|
||||
) duplicates
|
||||
ORDER BY `date` DESC, `timestamp` DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
COMMIT;
|
||||
@@ -0,0 +1,120 @@
|
||||
-- Update table and column profile timestamps to be in milliseconds
|
||||
UPDATE entity_extension_time_series
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.timestamp'),
|
||||
'$.timestamp',
|
||||
JSON_EXTRACT(json, '$.timestamp') * 1000
|
||||
)
|
||||
WHERE
|
||||
extension in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
|
||||
;
|
||||
|
||||
START TRANSACTION;
|
||||
-- Create report data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS report_data_time_series (
|
||||
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
|
||||
date DATE GENERATED ALWAYS AS (FROM_UNIXTIME((json ->> '$.timestamp') DIV 1000)) NOT NULL,
|
||||
INDEX report_data_time_series_point_ts (timestamp),
|
||||
INDEX report_data_time_series_date (date)
|
||||
);
|
||||
|
||||
INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension = 'reportData.reportDataResult';
|
||||
COMMIT;
|
||||
|
||||
START TRANSACTION;
|
||||
-- Create profiler data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS profiler_data_time_series (
|
||||
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
operation VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.operation') NULL,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
|
||||
UNIQUE profiler_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, operation, timestamp),
|
||||
INDEX profiler_data_time_series_combined_id_ts (extension, timestamp)
|
||||
);
|
||||
|
||||
INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series
|
||||
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
|
||||
COMMIT;
|
||||
|
||||
START TRANSACTION;
|
||||
-- Create data quality data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
|
||||
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
|
||||
UNIQUE data_quality_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, timestamp),
|
||||
INDEX data_quality_data_time_series_combined_id_ts (extension, timestamp)
|
||||
);
|
||||
|
||||
INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series
|
||||
WHERE extension = 'testCase.testCaseResult';
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension = 'testCase.testCaseResult';
|
||||
COMMIT;
|
||||
|
||||
ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin,
|
||||
ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
|
||||
ALTER TABLE field_relationship MODIFY COLUMN fromFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN toFQNHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE thread_entity MODIFY COLUMN entityLink VARCHAR(3072) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL;
|
||||
ALTER TABLE event_subscription_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE ingestion_pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE bot_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE user_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE team_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE role_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE policy_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE classification MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE tag MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE tag_usage MODIFY COLUMN tagFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN targetFQNHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE glossary_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE glossary_term_entity MODIFY fqnHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE type_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE web_analytic_event MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE data_insight_chart MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE kpi_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE test_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE test_connection_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE test_suite MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE test_case MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE dbservice_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE messaging_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE metadata_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE pipeline_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE dashboard_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE mlmodel_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE storage_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE database_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE database_schema_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE table_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE dashboard_data_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE dashboard_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE chart_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE ml_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE metric_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE query_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
|
||||
ALTER TABLE report_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE storage_container_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
ALTER TABLE topic_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
|
||||
@@ -0,0 +1,53 @@
|
||||
BEGIN;
|
||||
-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
|
||||
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> 'id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> 'reportDataType' = 'WebAnalyticEntityViewReportData'
|
||||
) duplicates
|
||||
ORDER BY date DESC, timestamp DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
|
||||
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> 'id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> 'reportDataType' = 'EntityReportData'
|
||||
) duplicates
|
||||
ORDER BY date DESC, timestamp DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
|
||||
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
|
||||
select ids FROM (
|
||||
SELECT
|
||||
(json ->> 'id') AS ids,
|
||||
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
|
||||
FROM report_data_time_series rdts
|
||||
WHERE json ->> 'reportDataType' = 'WebAnalyticUserActivityReportData'
|
||||
) duplicates
|
||||
ORDER BY date DESC, timestamp DESC
|
||||
) as dense_ranked
|
||||
WHERE denseRank != 1
|
||||
);
|
||||
COMMIT;
|
||||
@@ -0,0 +1,83 @@
|
||||
-- Update table and column profile timestamps to be in milliseconds
|
||||
UPDATE entity_extension_time_series
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{timestamp}',
|
||||
to_jsonb(cast(json#>'{timestamp}' as int8) *1000)
|
||||
)
|
||||
WHERE
|
||||
extension in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
|
||||
;
|
||||
|
||||
BEGIN;
|
||||
-- Run the following SQL to update the schema in a transaction
|
||||
-- Create report data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS report_data_time_series (
|
||||
entityFQNHash VARCHAR(768),
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS report_data_time_series_point_ts ON report_data_time_series (timestamp);
|
||||
|
||||
INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension = 'reportData.reportDataResult';
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
-- Create profiler data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS profiler_data_time_series (
|
||||
entityFQNHash VARCHAR(768),
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
operation VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'operation')::text) STORED NULL,
|
||||
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
|
||||
CONSTRAINT profiler_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, operation, timestamp)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS profiler_data_time_series_combined_id_ts ON profiler_data_time_series (extension, timestamp);
|
||||
|
||||
INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series
|
||||
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
-- Create profiler data time series table and move data from entity_extension_time_series
|
||||
CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
|
||||
entityFQNHash VARCHAR(768),
|
||||
extension VARCHAR(256) NOT NULL,
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
|
||||
CONSTRAINT data_quality_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, timestamp)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS data_quality_data_time_series_combined_id_ts ON data_quality_data_time_series (extension, timestamp);
|
||||
|
||||
INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
|
||||
SELECT entityFQNHash, extension, jsonSchema, json
|
||||
FROM entity_extension_time_series
|
||||
WHERE extension = 'testCase.testCaseResult';
|
||||
|
||||
DELETE FROM entity_extension_time_series
|
||||
WHERE extension = 'testCase.testCaseResult';
|
||||
COMMIT;
|
||||
|
||||
ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQNHash TYPE VARCHAR(768), ALTER COLUMN jsonSchema TYPE VARCHAR(50) , ALTER COLUMN extension TYPE VARCHAR(100) ,
|
||||
ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
|
||||
ALTER TABLE field_relationship ALTER COLUMN fromFQNHash TYPE VARCHAR(768), ALTER COLUMN toFQNHash TYPE VARCHAR(768);
|
||||
ALTER TABLE thread_entity ALTER COLUMN entityLink TYPE VARCHAR(3072);
|
||||
ALTER TABLE tag_usage ALTER COLUMN tagFQNHash TYPE VARCHAR(768), ALTER COLUMN targetFQNHash TYPE VARCHAR(768);
|
||||
ALTER TABLE test_suite ALTER COLUMN fqnHash TYPE VARCHAR(768);
|
||||
@@ -0,0 +1,63 @@
|
||||
-- Migration script to restructure Databricks connection configuration
|
||||
-- Move 'token' field from connection.config.token to connection.config.authType.token
|
||||
UPDATE dbservice_entity
|
||||
SET
|
||||
json = JSON_SET (
|
||||
JSON_REMOVE (json, '$.connection.config.token'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT (
|
||||
'token',
|
||||
JSON_EXTRACT (json, '$.connection.config.token')
|
||||
)
|
||||
)
|
||||
WHERE
|
||||
serviceType in ('Databricks', 'UnityCatalog')
|
||||
AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.token');
|
||||
|
||||
-- Migration to remove .png extension from appScreenshots in apps tables
|
||||
-- Part of fixing appScreenshots extension handling change
|
||||
|
||||
-- Update apps_marketplace table - remove .png extension from appScreenshots only
|
||||
UPDATE apps_marketplace
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.appScreenshots',
|
||||
JSON_EXTRACT(
|
||||
REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
|
||||
'$'
|
||||
)
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
|
||||
AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';
|
||||
|
||||
-- Update installed_apps table - remove .png extension from appScreenshots only
|
||||
UPDATE installed_apps
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.appScreenshots',
|
||||
JSON_EXTRACT(
|
||||
REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
|
||||
'$'
|
||||
)
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
|
||||
AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';
|
||||
|
||||
-- Update apps_data_store table - remove .png extension from appScreenshots only
|
||||
UPDATE apps_data_store
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.appScreenshots',
|
||||
JSON_EXTRACT(
|
||||
REPLACE(JSON_EXTRACT(json, '$.appScreenshots'), '.png"', '"'),
|
||||
'$'
|
||||
)
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') IS NOT NULL
|
||||
AND JSON_LENGTH(JSON_EXTRACT(json, '$.appScreenshots')) > 0
|
||||
AND JSON_EXTRACT(json, '$.appScreenshots') LIKE '%.png%';
|
||||
@@ -0,0 +1,83 @@
|
||||
-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration
|
||||
UPDATE openmetadata_settings
|
||||
SET json = JSON_ARRAY_APPEND(
|
||||
json,
|
||||
'$.entitySemantics',
|
||||
JSON_OBJECT(
|
||||
'name', 'Data Product Domain Validation',
|
||||
'description', 'Validates that Data Products assigned to an entity match the entity''s domains.',
|
||||
'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}',
|
||||
'enabled', true,
|
||||
'provider', 'system'
|
||||
)
|
||||
)
|
||||
WHERE configType = 'entityRulesSettings'
|
||||
AND JSON_EXTRACT(json, '$.entitySemantics') IS NOT NULL
|
||||
AND NOT JSON_CONTAINS(
|
||||
JSON_EXTRACT(json, '$.entitySemantics[*].name'),
|
||||
JSON_QUOTE('Data Product Domain Validation')
|
||||
);
|
||||
|
||||
-- Add virtual column for customUnitOfMeasurement
|
||||
ALTER TABLE metric_entity
|
||||
ADD COLUMN customUnitOfMeasurement VARCHAR(256)
|
||||
GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.customUnitOfMeasurement'))) VIRTUAL;
|
||||
-- Add index on the virtual column
|
||||
CREATE INDEX idx_metric_custom_unit ON metric_entity(customUnitOfMeasurement);
|
||||
|
||||
-- Fetch updated searchSettings
|
||||
DELETE FROM openmetadata_settings WHERE configType = 'searchSettings';
|
||||
|
||||
-- Create notification_template_entity table following OpenMetadata patterns
|
||||
CREATE TABLE IF NOT EXISTS notification_template_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.id'))) STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.name'))) VIRTUAL NOT NULL,
|
||||
fqnHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.updatedAt'))) VIRTUAL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.updatedBy'))) VIRTUAL NOT NULL,
|
||||
deleted TINYINT(1) GENERATED ALWAYS AS (json_extract(json, '$.deleted')) VIRTUAL,
|
||||
provider VARCHAR(32) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.provider'))) VIRTUAL,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY fqnHash (fqnHash),
|
||||
INDEX idx_notification_template_name (name),
|
||||
INDEX idx_notification_template_provider (provider)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
-- Optimize table listing queries by indexing the schema hash prefix
|
||||
ALTER TABLE table_entity
|
||||
ADD COLUMN databaseSchemaHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin
|
||||
GENERATED ALWAYS AS (SUBSTRING_INDEX(fqnHash, '.', 3)) STORED;
|
||||
|
||||
CREATE INDEX idx_table_entity_schema_listing
|
||||
ON table_entity (deleted, databaseSchemaHash, name, id);
|
||||
|
||||
-- Optimize stored procedure listing queries by indexing the schema hash prefix
|
||||
ALTER TABLE stored_procedure_entity
|
||||
ADD COLUMN databaseSchemaHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin
|
||||
GENERATED ALWAYS AS (SUBSTRING_INDEX(fqnHash, '.', 3)) STORED;
|
||||
|
||||
CREATE INDEX idx_stored_procedure_schema_listing
|
||||
ON stored_procedure_entity (deleted, databaseSchemaHash, name, id);
|
||||
|
||||
-- Recognizer Feedback Storage
|
||||
-- Store user feedback on auto-applied tags to improve recognition accuracy
|
||||
CREATE TABLE IF NOT EXISTS recognizer_feedback_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.id'))) STORED NOT NULL,
|
||||
entityLink VARCHAR(512) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.entityLink'))) VIRTUAL NOT NULL,
|
||||
tagFQN VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.tagFQN'))) VIRTUAL NOT NULL,
|
||||
feedbackType VARCHAR(50) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.feedbackType'))) VIRTUAL NOT NULL,
|
||||
status VARCHAR(20) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.status'))) VIRTUAL,
|
||||
createdBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.createdBy'))) VIRTUAL NOT NULL,
|
||||
createdAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.createdAt'))) VIRTUAL NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
INDEX idx_feedback_entity (entityLink),
|
||||
INDEX idx_feedback_tag (tagFQN),
|
||||
INDEX idx_feedback_status (status),
|
||||
INDEX idx_feedback_created (createdAt)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN reason TEXT;
|
||||
@@ -0,0 +1,51 @@
|
||||
-- Migration script to restructure Databricks connection configuration
|
||||
-- Move 'token' field from connection.config.token to connection.config.authType.token
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #- '{connection,config,token}',
|
||||
'{connection,config,authType}',
|
||||
jsonb_build_object('token', json #> '{connection,config,token}'),
|
||||
true
|
||||
)
|
||||
WHERE serviceType in ('Databricks', 'UnityCatalog')
|
||||
AND jsonb_exists(json -> 'connection' -> 'config', 'token');
|
||||
|
||||
-- Migration to remove .png extension from appScreenshots in apps tables
|
||||
-- Part of fixing appScreenshots extension handling change
|
||||
|
||||
-- Update apps_marketplace table - remove .png extension from appScreenshots only
|
||||
UPDATE apps_marketplace
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{appScreenshots}',
|
||||
REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND json -> 'appScreenshots' IS NOT NULL
|
||||
AND jsonb_array_length(json -> 'appScreenshots') > 0
|
||||
AND (json -> 'appScreenshots')::text LIKE '%.png%';
|
||||
|
||||
-- Update installed_apps table - remove .png extension from appScreenshots only
|
||||
UPDATE installed_apps
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{appScreenshots}',
|
||||
REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND json -> 'appScreenshots' IS NOT NULL
|
||||
AND jsonb_array_length(json -> 'appScreenshots') > 0
|
||||
AND (json -> 'appScreenshots')::text LIKE '%.png%';
|
||||
|
||||
-- Update apps_data_store table - remove .png extension from appScreenshots only
|
||||
UPDATE apps_data_store
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{appScreenshots}',
|
||||
REPLACE((json -> 'appScreenshots')::text, '.png"', '"')::jsonb
|
||||
)
|
||||
WHERE json IS NOT NULL
|
||||
AND json -> 'appScreenshots' IS NOT NULL
|
||||
AND json_array_length((json -> 'appScreenshots')::json) > 0
|
||||
AND (json -> 'appScreenshots')::text LIKE '%.png%';
|
||||
@@ -0,0 +1,104 @@
|
||||
-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration
|
||||
UPDATE openmetadata_settings
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{entitySemantics}',
|
||||
(json->'entitySemantics') || jsonb_build_object(
|
||||
'name', 'Data Product Domain Validation',
|
||||
'description', 'Validates that Data Products assigned to an entity match the entity''s domains.',
|
||||
'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}',
|
||||
'enabled', true,
|
||||
'provider', 'system'
|
||||
)::jsonb,
|
||||
true
|
||||
)
|
||||
WHERE configtype = 'entityRulesSettings'
|
||||
AND json->'entitySemantics' IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM jsonb_array_elements(json->'entitySemantics') AS rule
|
||||
WHERE rule->>'name' = 'Data Product Domain Validation'
|
||||
);
|
||||
|
||||
-- Add generated column for customUnitOfMeasurement
|
||||
ALTER TABLE metric_entity
|
||||
ADD COLUMN customUnitOfMeasurement VARCHAR(256)
|
||||
GENERATED ALWAYS AS ((json->>'customUnitOfMeasurement')::VARCHAR(256)) STORED;
|
||||
-- Add index on the column
|
||||
CREATE INDEX idx_metric_custom_unit ON metric_entity(customUnitOfMeasurement);
|
||||
|
||||
-- Fetch updated searchSettings
|
||||
DELETE FROM openmetadata_settings WHERE configType = 'searchSettings';
|
||||
|
||||
-- Create notification_template_entity table following OpenMetadata patterns
|
||||
CREATE TABLE IF NOT EXISTS notification_template_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
provider VARCHAR(32) GENERATED ALWAYS AS (json ->> 'provider') STORED,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_template_name ON notification_template_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_template_provider ON notification_template_entity(provider);
|
||||
|
||||
-- Optimize table listing queries by indexing the schema hash prefix
|
||||
ALTER TABLE table_entity
|
||||
ADD COLUMN IF NOT EXISTS databaseSchemaHash VARCHAR(768)
|
||||
GENERATED ALWAYS AS (
|
||||
rtrim(
|
||||
split_part(fqnhash, '.', 1) || '.' ||
|
||||
split_part(fqnhash, '.', 2) || '.' ||
|
||||
split_part(fqnhash, '.', 3),
|
||||
'.'
|
||||
)
|
||||
) STORED;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_table_entity_schema_listing
|
||||
ON table_entity (deleted, databaseSchemaHash, name, id);
|
||||
|
||||
-- Optimize stored procedure listing queries by indexing the schema hash prefix
|
||||
ALTER TABLE stored_procedure_entity
|
||||
ADD COLUMN IF NOT EXISTS databaseSchemaHash VARCHAR(768)
|
||||
GENERATED ALWAYS AS (
|
||||
rtrim(
|
||||
split_part(fqnhash, '.', 1) || '.' ||
|
||||
split_part(fqnhash, '.', 2) || '.' ||
|
||||
split_part(fqnhash, '.', 3),
|
||||
'.'
|
||||
)
|
||||
) STORED;
|
||||
|
||||
DROP INDEX IF EXISTS idx_stored_procedure_entity_deleted_name_id;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_stored_procedure_schema_listing
|
||||
ON stored_procedure_entity (deleted, databaseSchemaHash, name, id);
|
||||
|
||||
|
||||
-- Recognizer Feedback Storage
|
||||
-- Store user feedback on auto-applied tags to improve recognition accuracy
|
||||
CREATE TABLE IF NOT EXISTS recognizer_feedback_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
entityLink VARCHAR(512) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL,
|
||||
tagFQN VARCHAR(256) GENERATED ALWAYS AS (json ->> 'tagFQN') STORED NOT NULL,
|
||||
feedbackType VARCHAR(50) GENERATED ALWAYS AS (json ->> 'feedbackType') STORED NOT NULL,
|
||||
status VARCHAR(20) GENERATED ALWAYS AS (json ->> 'status') STORED,
|
||||
createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'createdBy') STORED NOT NULL,
|
||||
createdAt BIGINT GENERATED ALWAYS AS ((json ->> 'createdAt')::bigint) STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_feedback_entity ON recognizer_feedback_entity(entityLink);
|
||||
CREATE INDEX IF NOT EXISTS idx_feedback_tag ON recognizer_feedback_entity(tagFQN);
|
||||
CREATE INDEX IF NOT EXISTS idx_feedback_status ON recognizer_feedback_entity(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_feedback_created ON recognizer_feedback_entity(createdAt);
|
||||
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN reason TEXT;
|
||||
@@ -0,0 +1,2 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1102
|
||||
@@ -0,0 +1,2 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1102
|
||||
@@ -0,0 +1,8 @@
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_INSERT(
|
||||
json,
|
||||
CONCAT('$.supportedDataTypes[', JSON_LENGTH(json, '$.supportedDataTypes'), ']'),
|
||||
'NUMERIC'
|
||||
)
|
||||
WHERE JSON_CONTAINS(json->'$.supportedDataTypes', '"NUMBER"')
|
||||
AND NOT JSON_CONTAINS(json->'$.supportedDataTypes', '"NUMERIC"');
|
||||
@@ -0,0 +1,9 @@
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '"NUMERIC"'::jsonb,
|
||||
true
|
||||
)
|
||||
WHERE json->'supportedDataTypes' @> '"NUMBER"'::jsonb
|
||||
AND NOT (json->'supportedDataTypes' @> '"NUMERIC"'::jsonb);
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Delete searchSettings to force reload from packaged searchSettings.json with field-based aggregations
|
||||
DELETE FROM openmetadata_settings WHERE configType='searchSettings';
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Delete searchSettings to force reload from packaged searchSettings.json with field-based aggregations
|
||||
DELETE FROM openmetadata_settings WHERE configType='searchSettings';
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Increase Flowable ACTIVITY_ID_ column size to support longer user-defined workflow node names
|
||||
ALTER TABLE ACT_RU_EVENT_SUBSCR MODIFY ACTIVITY_ID_ varchar(255);
|
||||
|
||||
-- Update workflow settings with new job acquisition interval settings
|
||||
UPDATE openmetadata_settings
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.executorConfiguration.asyncJobAcquisitionInterval', 60000,
|
||||
'$.executorConfiguration.timerJobAcquisitionInterval', 60000
|
||||
)
|
||||
WHERE configType = 'workflowSettings'
|
||||
AND JSON_EXTRACT(json, '$.executorConfiguration') IS NOT NULL;
|
||||
|
||||
-- TRUNCATE FLOWABLE HISTORY TABLES
|
||||
TRUNCATE TABLE ACT_HI_PROCINST;
|
||||
TRUNCATE TABLE ACT_HI_ACTINST;
|
||||
TRUNCATE TABLE ACT_HI_TASKINST;
|
||||
TRUNCATE TABLE ACT_HI_VARINST;
|
||||
TRUNCATE TABLE ACT_HI_DETAIL;
|
||||
TRUNCATE TABLE ACT_HI_COMMENT;
|
||||
TRUNCATE TABLE ACT_HI_ATTACHMENT;
|
||||
TRUNCATE TABLE ACT_HI_IDENTITYLINK;
|
||||
@@ -0,0 +1,28 @@
|
||||
-- Increase Flowable ACTIVITY_ID_ column size to support longer user-defined workflow node names
|
||||
ALTER TABLE ACT_RU_EVENT_SUBSCR ALTER COLUMN ACTIVITY_ID_ TYPE varchar(255);
|
||||
|
||||
-- Update workflow settings with new job acquisition interval settings
|
||||
UPDATE openmetadata_settings
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
json,
|
||||
'{executorConfiguration,asyncJobAcquisitionInterval}',
|
||||
'60000',
|
||||
true
|
||||
),
|
||||
'{executorConfiguration,timerJobAcquisitionInterval}',
|
||||
'60000',
|
||||
true
|
||||
)
|
||||
WHERE configtype = 'workflowSettings'
|
||||
AND json->'executorConfiguration' IS NOT NULL;
|
||||
|
||||
-- TRUNCATE FLOWABLE HISTORY TABLES
|
||||
TRUNCATE TABLE ACT_HI_PROCINST;
|
||||
TRUNCATE TABLE ACT_HI_ACTINST;
|
||||
TRUNCATE TABLE ACT_HI_TASKINST;
|
||||
TRUNCATE TABLE ACT_HI_VARINST;
|
||||
TRUNCATE TABLE ACT_HI_DETAIL;
|
||||
TRUNCATE TABLE ACT_HI_COMMENT;
|
||||
TRUNCATE TABLE ACT_HI_ATTACHMENT;
|
||||
TRUNCATE TABLE ACT_HI_IDENTITYLINK;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Migrate Airbyte connections from flat username/password to auth object structure
|
||||
UPDATE pipeline_service_entity
|
||||
SET
|
||||
json = JSON_SET (
|
||||
JSON_REMOVE (
|
||||
json,
|
||||
'$.connection.config.username',
|
||||
'$.connection.config.password'
|
||||
),
|
||||
'$.connection.config.auth',
|
||||
JSON_OBJECT (
|
||||
'username',
|
||||
JSON_UNQUOTE (
|
||||
JSON_EXTRACT (json, '$.connection.config.username')
|
||||
),
|
||||
'password',
|
||||
JSON_UNQUOTE (
|
||||
JSON_EXTRACT (json, '$.connection.config.password')
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE
|
||||
JSON_EXTRACT (json, '$.serviceType') = 'Airbyte';
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Migrate Airbyte connections from flat username/password to auth object structure
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(
|
||||
json #- '{connection,config,username}' #- '{connection,config,password}',
|
||||
'{connection,config,auth}',
|
||||
jsonb_build_object(
|
||||
'username', json#>>'{connection,config,username}',
|
||||
'password', json#>>'{connection,config,password}'
|
||||
),
|
||||
true
|
||||
)
|
||||
WHERE json->>'serviceType' = 'Airbyte';
|
||||
@@ -0,0 +1,9 @@
|
||||
-- TRUNCATE FLOWABLE HISTORY TABLES
|
||||
TRUNCATE TABLE ACT_HI_PROCINST;
|
||||
TRUNCATE TABLE ACT_HI_ACTINST;
|
||||
TRUNCATE TABLE ACT_HI_TASKINST;
|
||||
TRUNCATE TABLE ACT_HI_VARINST;
|
||||
TRUNCATE TABLE ACT_HI_DETAIL;
|
||||
TRUNCATE TABLE ACT_HI_COMMENT;
|
||||
TRUNCATE TABLE ACT_HI_ATTACHMENT;
|
||||
TRUNCATE TABLE ACT_HI_IDENTITYLINK;
|
||||
@@ -0,0 +1,9 @@
|
||||
-- TRUNCATE FLOWABLE HISTORY TABLES
|
||||
TRUNCATE TABLE ACT_HI_PROCINST;
|
||||
TRUNCATE TABLE ACT_HI_ACTINST;
|
||||
TRUNCATE TABLE ACT_HI_TASKINST;
|
||||
TRUNCATE TABLE ACT_HI_VARINST;
|
||||
TRUNCATE TABLE ACT_HI_DETAIL;
|
||||
TRUNCATE TABLE ACT_HI_COMMENT;
|
||||
TRUNCATE TABLE ACT_HI_ATTACHMENT;
|
||||
TRUNCATE TABLE ACT_HI_IDENTITYLINK;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Fix emailConfiguration templates field for users affected by v1.8.0 bug
|
||||
-- See commit: https://github.com/open-metadata/OpenMetadata/commit/a50cddf665544b0934ce79a539aecbb00bd541c8
|
||||
-- This migration ensures all emailConfiguration records have the correct 'openmetadata' value
|
||||
UPDATE openmetadata_settings
|
||||
SET json = JSON_SET(json, '$.templates', 'openmetadata')
|
||||
WHERE configType = 'emailConfiguration';
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Fix emailConfiguration templates field for users affected by v1.8.0 bug
|
||||
-- See commit: https://github.com/open-metadata/OpenMetadata/commit/a50cddf665544b0934ce79a539aecbb00bd541c8
|
||||
-- This migration ensures all emailConfiguration records have the correct 'openmetadata' value
|
||||
UPDATE openmetadata_settings
|
||||
SET json = jsonb_set(json, '{templates}', '"openmetadata"')
|
||||
WHERE configType = 'emailConfiguration';
|
||||
@@ -0,0 +1,104 @@
|
||||
-- Correct the table diff test definition
|
||||
-- This is to include the new parameter table2.keyColumns
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.parameterDefinition',
|
||||
JSON_ARRAY(
|
||||
JSON_OBJECT(
|
||||
'name', 'keyColumns',
|
||||
'displayName', 'Table 1''s key columns',
|
||||
'description', 'The columns to use as the key for the comparison. If not provided, it will be resolved from the primary key or unique columns. The tuples created from the key columns must be unique.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'table2',
|
||||
'displayName', 'Table 2',
|
||||
'description', 'Fully qualified name of the table to compare against.',
|
||||
'dataType', 'STRING',
|
||||
'required', true
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'table2.keyColumns',
|
||||
'displayName', 'Table 2''s key columns',
|
||||
'description', 'The columns in table 2 to use as comparison. If not provided, it will default to `Key Columns`, risking errors if the key columns'' names have changed.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'threshold',
|
||||
'displayName', 'Threshold',
|
||||
'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
|
||||
'dataType', 'NUMBER',
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'useColumns',
|
||||
'displayName', 'Use Columns',
|
||||
'description', 'Limits the scope of the test to this list of columns. If not provided, all columns will be used except the key columns.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'where',
|
||||
'displayName', 'SQL Where Clause',
|
||||
'description', 'Use this where clause to filter the rows to compare.',
|
||||
'dataType', 'STRING',
|
||||
'required', false
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'name', 'caseSensitiveColumns',
|
||||
'displayName', 'Case sensitive columns',
|
||||
'description', 'Use case sensitivity when comparing the columns.',
|
||||
'dataType', 'BOOLEAN',
|
||||
'required', false
|
||||
)
|
||||
),
|
||||
'$.version',
|
||||
0.2
|
||||
)
|
||||
WHERE name = 'tableDiff';
|
||||
|
||||
-- Update DataRetentionApplication add profileDataRetentionPeriod and testCaseResultsRetentionPeriod
|
||||
UPDATE installed_apps
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.appConfiguration.testCaseResultsRetentionPeriod', 1440,
|
||||
'$.appConfiguration.profileDataRetentionPeriod', 1440
|
||||
)
|
||||
WHERE JSON_EXTRACT(json, '$.name') = 'DataRetentionApplication';
|
||||
|
||||
UPDATE notification_template_entity
|
||||
SET json = JSON_REMOVE(json, '$.defaultTemplateChecksum')
|
||||
WHERE JSON_CONTAINS_PATH(json, 'one', '$.defaultTemplateChecksum');
|
||||
|
||||
-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAIQualityAgentApplication and CollateAITierAgentApplication
|
||||
UPDATE apps_marketplace
|
||||
SET json = JSON_SET(
|
||||
JSON_SET(
|
||||
json,
|
||||
'$.appType',
|
||||
'external'
|
||||
),
|
||||
'$.sourcePythonClass',
|
||||
'metadata.applications.dynamic_agent.app.DynamicAgentApp'
|
||||
)
|
||||
WHERE json->>'$.name' IN ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
|
||||
AND json->>'$.appType' = 'internal' AND json->>'$.sourcePythonClass' IS NULL;
|
||||
|
||||
-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAITierAgentApplication and CollateAIQualityAgentApplication
|
||||
UPDATE installed_apps
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.appType', 'external',
|
||||
'$.sourcePythonClass', 'metadata.applications.dynamic_agent.app.DynamicAgentApp'
|
||||
)
|
||||
WHERE json->>'$.name' IN ('CollateAITierAgentApplication', 'CollateAIQualityAgentApplication')
|
||||
AND json->>'$.appType' = 'internal' AND json->>'$.sourcePythonClass' IS NULL;
|
||||
|
||||
-- Remove bot form App entity
|
||||
UPDATE installed_apps SET json = JSON_REMOVE(json, '$.bot') WHERE JSON_CONTAINS_PATH(json, 'one', '$.bot');
|
||||
|
||||
-- Remove SearchIndexingApplication past runs
|
||||
delete from apps_extension_time_series where appname = 'SearchIndexingApplication';
|
||||
@@ -0,0 +1,140 @@
|
||||
-- Test Case Dimension Results Time Series Table
|
||||
CREATE TABLE IF NOT EXISTS test_case_dimension_results_time_series (
|
||||
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
extension VARCHAR(256) NOT NULL DEFAULT 'testCase.dimensionResult',
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.id'))) STORED NOT NULL,
|
||||
testCaseResultId VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseResultId'))) STORED NOT NULL,
|
||||
dimensionKey VARCHAR(512) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.dimensionKey'))) STORED NOT NULL,
|
||||
dimensionName VARCHAR(256) GENERATED ALWAYS AS (SUBSTRING_INDEX(json_unquote(json_extract(json,'$.dimensionKey')), '=', 1)) STORED,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.timestamp'))) STORED NOT NULL,
|
||||
testCaseStatus VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(json,'$.testCaseStatus'))) STORED,
|
||||
UNIQUE KEY test_case_dimension_results_unique_constraint (entityFQNHash, dimensionKey, timestamp),
|
||||
INDEX test_case_dimension_results_main (entityFQNHash, timestamp, dimensionKey),
|
||||
INDEX test_case_dimension_results_dimension_name (entityFQNHash, dimensionName, timestamp),
|
||||
INDEX test_case_dimension_results_result_id (testCaseResultId),
|
||||
INDEX test_case_dimension_results_ts (timestamp)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
-- Add impersonatedBy column to all entity tables for tracking bot impersonation
|
||||
-- This column stores which bot performed an action on behalf of a user
|
||||
|
||||
ALTER TABLE api_collection_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE api_endpoint_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE api_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE bot_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE chart_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE dashboard_data_model_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE dashboard_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE dashboard_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE data_contract_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE data_product_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE database_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE database_schema_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE dbservice_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE directory_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE domain_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE drive_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE event_subscription_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE file_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE glossary_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE glossary_term_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE ingestion_pipeline_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE kpi_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE messaging_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE metadata_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE metric_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE ml_model_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE mlmodel_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE notification_template_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE persona_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE pipeline_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE pipeline_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE policy_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE query_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE report_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE role_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE search_index_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE search_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE security_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE spreadsheet_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE storage_container_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE storage_service_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE stored_procedure_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE table_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE team_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE thread_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE topic_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE type_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE user_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE workflow_definition_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
ALTER TABLE worksheet_entity ADD COLUMN impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.impersonatedBy'))) VIRTUAL;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.testPlatforms',
|
||||
CAST(REPLACE(
|
||||
JSON_EXTRACT(json, '$.testPlatforms'),
|
||||
'"DBT"',
|
||||
'"dbt"'
|
||||
) AS JSON)
|
||||
)
|
||||
WHERE JSON_CONTAINS(json, '"DBT"', '$.testPlatforms');
|
||||
|
||||
-- Performance optimization for tag_usage prefix queries
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN targetfqnhash_lower VARCHAR(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
|
||||
GENERATED ALWAYS AS (CONVERT(LOWER(targetFQNHash) USING utf8mb4) COLLATE utf8mb4_unicode_ci) STORED;
|
||||
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN tagfqn_lower VARCHAR(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
|
||||
GENERATED ALWAYS AS (CONVERT(LOWER(tagFQN) USING utf8mb4) COLLATE utf8mb4_unicode_ci) STORED;
|
||||
|
||||
-- Create indexes on the new columns
|
||||
CREATE INDEX idx_targetfqnhash_lower ON tag_usage (targetfqnhash_lower(255));
|
||||
CREATE INDEX idx_tagfqn_lower ON tag_usage (tagfqn_lower(255));
|
||||
|
||||
-- PRIMARY INDEX: For targetFQNHash prefix searches (LIKE 'prefix%')
|
||||
-- Using prefix indexing for large VARCHAR columns to stay within MySQL's 3072 byte limit
|
||||
CREATE INDEX idx_tag_usage_target_prefix_composite
|
||||
ON tag_usage (source, targetfqnhash_lower(255), state, tagFQN(255), labelType);
|
||||
|
||||
-- For exact match queries
|
||||
CREATE INDEX idx_tag_usage_target_exact_composite
|
||||
ON tag_usage (source, targetFQNHash(255), state, tagFQN(255), labelType);
|
||||
|
||||
-- For tagFQN prefix searches
|
||||
CREATE INDEX idx_tag_usage_tagfqn_prefix_composite
|
||||
ON tag_usage (source, tagfqn_lower(255), state, targetFQNHash(255), labelType);
|
||||
|
||||
-- For JOIN operations
|
||||
CREATE INDEX idx_tag_usage_join_composite
|
||||
ON tag_usage (tagFQNHash(255), source, state, targetFQNHash(255), tagFQN(255), labelType);
|
||||
|
||||
CREATE FULLTEXT INDEX ft_tag_usage_targetfqn
|
||||
ON tag_usage(targetFQNHash);
|
||||
|
||||
-- Add index for efficient bulk term count queries
|
||||
-- The bulkGetTermCounts query uses: WHERE classificationHash IN (...) AND deleted = FALSE
|
||||
CREATE INDEX idx_tag_classification_deleted
|
||||
ON tag (classificationHash, deleted);
|
||||
|
||||
-- Create new composite indexes with deleted column
|
||||
-- MySQL doesn't support INCLUDE clause, so we put all columns in the index
|
||||
CREATE INDEX idx_entity_relationship_from_deleted
|
||||
ON entity_relationship(fromId, fromEntity, relation, deleted, toId, toEntity);
|
||||
|
||||
CREATE INDEX idx_entity_relationship_to_deleted
|
||||
ON entity_relationship(toId, toEntity, relation, deleted, fromId, fromEntity);
|
||||
|
||||
-- Index for queries that filter by fromEntity/toEntity
|
||||
CREATE INDEX idx_entity_relationship_from_typed
|
||||
ON entity_relationship(toId, toEntity, relation, fromEntity, deleted, fromId);
|
||||
|
||||
-- Index for bidirectional lookups
|
||||
CREATE INDEX idx_entity_relationship_bidirectional
|
||||
ON entity_relationship(fromId, toId, relation, deleted);
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
-- Correct the table diff test definition
|
||||
-- This is to include the new parameter table2.keyColumns
|
||||
UPDATE test_definition
|
||||
SET json = json::jsonb || json_build_object(
|
||||
'parameterDefinition', jsonb_build_array(
|
||||
jsonb_build_object(
|
||||
'name', 'keyColumns',
|
||||
'displayName', 'Table 1''s key Columns',
|
||||
'description', 'The columns to use as the key for the comparison. If not provided, it will be resolved from the primary key or unique columns. The tuples created from the key columns must be unique.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'table2',
|
||||
'displayName', 'Table 2',
|
||||
'description', 'Fully qualified name of the table to compare against.',
|
||||
'dataType', 'STRING',
|
||||
'required', true
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'table2.keyColumns',
|
||||
'displayName', 'Table 2''s key columns',
|
||||
'description', 'The columns in table 2 to use as comparison. If not provided, it will default to `Key Columns`, risking errors if the key columns'' names have changed.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'threshold',
|
||||
'displayName', 'Threshold',
|
||||
'description', 'Threshold to use to determine if the test passes or fails (defaults to 0).',
|
||||
'dataType', 'NUMBER',
|
||||
'required', false
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'useColumns',
|
||||
'displayName', 'Use Columns',
|
||||
'description', 'Limits the scope of the test to this list of columns. If not provided, all columns will be used except the key columns.',
|
||||
'dataType', 'ARRAY',
|
||||
'required', false
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'where',
|
||||
'displayName', 'SQL Where Clause',
|
||||
'description', 'Use this where clause to filter the rows to compare.',
|
||||
'dataType', 'STRING',
|
||||
'required', false
|
||||
),
|
||||
jsonb_build_object(
|
||||
'name', 'caseSensitiveColumns',
|
||||
'displayName', 'Case sensitive columns',
|
||||
'description', 'Use case sensitivity when comparing the columns.',
|
||||
'dataType', 'BOOLEAN',
|
||||
'required', false
|
||||
)
|
||||
),
|
||||
'version', 0.2
|
||||
)::jsonb
|
||||
WHERE name = 'tableDiff';
|
||||
|
||||
-- Update DataRetentionApplication add profileDataRetentionPeriod and testCaseResultsRetentionPeriod
|
||||
UPDATE installed_apps
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
json,
|
||||
'{appConfiguration, testCaseResultsRetentionPeriod}',
|
||||
'1440'
|
||||
),
|
||||
'{appConfiguration, profileDataRetentionPeriod}',
|
||||
'1440'
|
||||
)
|
||||
WHERE json->>'name' = 'DataRetentionApplication';
|
||||
|
||||
UPDATE notification_template_entity
|
||||
SET json = json::jsonb - 'defaultTemplateChecksum';
|
||||
|
||||
-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAIQualityAgentApplication and CollateAITierAgentApplication
|
||||
UPDATE apps_marketplace
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
json::jsonb,
|
||||
'{appType}',
|
||||
'"external"'
|
||||
),
|
||||
'{sourcePythonClass}',
|
||||
'"metadata.applications.dynamic_agent.app.DynamicAgentApp"'
|
||||
)
|
||||
WHERE json->>'name' IN ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
|
||||
AND json->>'appType' = 'internal' AND json->>'sourcePythonClass' IS NULL;
|
||||
|
||||
-- Update appType from 'internal' to 'external' and add sourcePythonClass for CollateAITierAgentApplication and CollateAIQualityAgentApplication
|
||||
UPDATE installed_apps
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
json::jsonb,
|
||||
'{appType}',
|
||||
'"external"'
|
||||
),
|
||||
'{sourcePythonClass}',
|
||||
'"metadata.applications.dynamic_agent.app.DynamicAgentApp"'
|
||||
)
|
||||
WHERE json->>'name' IN ('CollateAIQualityAgentApplication', 'CollateAITierAgentApplication')
|
||||
AND json->>'appType' = 'internal' AND json->>'sourcePythonClass' IS NULL;
|
||||
|
||||
-- Remove bot form App entity
|
||||
UPDATE installed_apps SET json = json - 'bot';
|
||||
|
||||
-- Remove SearchIndexingApplication past runs
|
||||
delete from apps_extension_time_series where appname = 'SearchIndexingApplication';
|
||||
@@ -0,0 +1,168 @@
|
||||
-- Test Case Dimension Results Time Series Table
|
||||
CREATE TABLE IF NOT EXISTS test_case_dimension_results_time_series (
|
||||
entityFQNHash VARCHAR(768) COLLATE "C" NOT NULL,
|
||||
extension VARCHAR(256) NOT NULL DEFAULT 'testCase.dimensionResult',
|
||||
jsonSchema VARCHAR(256) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
|
||||
testCaseResultId VARCHAR(36) GENERATED ALWAYS AS (json ->> 'testCaseResultId') STORED NOT NULL,
|
||||
dimensionKey VARCHAR(512) GENERATED ALWAYS AS (json ->> 'dimensionKey') STORED NOT NULL,
|
||||
dimensionName VARCHAR(256) GENERATED ALWAYS AS (SPLIT_PART(json ->> 'dimensionKey', '=', 1)) STORED,
|
||||
timestamp BIGINT GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
|
||||
testCaseStatus VARCHAR(36) GENERATED ALWAYS AS (json ->> 'testCaseStatus') STORED,
|
||||
CONSTRAINT test_case_dimension_results_unique_constraint UNIQUE (entityFQNHash, dimensionKey, timestamp)
|
||||
);
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX IF NOT EXISTS test_case_dimension_results_main ON test_case_dimension_results_time_series (entityFQNHash, timestamp, dimensionKey);
|
||||
CREATE INDEX IF NOT EXISTS test_case_dimension_results_dimension_name ON test_case_dimension_results_time_series (entityFQNHash, dimensionName, timestamp);
|
||||
CREATE INDEX IF NOT EXISTS test_case_dimension_results_result_id ON test_case_dimension_results_time_series (testCaseResultId);
|
||||
CREATE INDEX IF NOT EXISTS test_case_dimension_results_ts ON test_case_dimension_results_time_series (timestamp);
|
||||
-- Add impersonatedBy column to all entity tables for tracking bot impersonation
|
||||
-- This column stores which bot performed an action on behalf of a user
|
||||
|
||||
ALTER TABLE api_collection_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE api_endpoint_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE api_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE bot_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE chart_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE dashboard_data_model_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE dashboard_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE dashboard_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE data_contract_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE data_product_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE database_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE database_schema_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE directory_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE domain_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE drive_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE event_subscription_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE file_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE glossary_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE glossary_term_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE ingestion_pipeline_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE kpi_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE messaging_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE metadata_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE metric_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE ml_model_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE mlmodel_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE notification_template_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE persona_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE pipeline_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE pipeline_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE policy_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE query_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE report_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE role_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE search_index_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE search_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE security_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE spreadsheet_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE storage_container_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE storage_service_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE stored_procedure_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE table_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE team_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE thread_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE topic_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE type_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE user_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE workflow_definition_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
ALTER TABLE worksheet_entity ADD COLUMN IF NOT EXISTS impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'impersonatedBy') STORED;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{testPlatforms}',
|
||||
REPLACE(
|
||||
(json::jsonb -> 'testPlatforms')::text,
|
||||
'"DBT"',
|
||||
'"dbt"'
|
||||
)::jsonb
|
||||
)::json
|
||||
WHERE json::jsonb -> 'testPlatforms' @> '"DBT"'::jsonb;
|
||||
|
||||
-- Performance optimization for tag_usage prefix queries
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN IF NOT EXISTS targetfqnhash_lower text
|
||||
GENERATED ALWAYS AS (lower(targetFQNHash)) STORED;
|
||||
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN IF NOT EXISTS tagfqn_lower text
|
||||
GENERATED ALWAYS AS (lower(tagFQN)) STORED;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_target_prefix_covering
|
||||
ON tag_usage (source, targetfqnhash_lower text_pattern_ops)
|
||||
INCLUDE (tagFQN, labelType, state)
|
||||
WHERE state = 1; -- Only active tags
|
||||
|
||||
-- For exact match queries on targetFQNHash
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_target_exact
|
||||
ON tag_usage (source, targetFQNHash, state)
|
||||
INCLUDE (tagFQN, labelType);
|
||||
|
||||
-- For tagFQN prefix searches if needed
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_tagfqn_prefix_covering
|
||||
ON tag_usage (source, tagfqn_lower text_pattern_ops)
|
||||
INCLUDE (targetFQNHash, labelType, state)
|
||||
WHERE state = 1;
|
||||
|
||||
-- For JOIN operations with classification and tag tables
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_usage_join_source
|
||||
ON tag_usage (tagFQNHash, source)
|
||||
INCLUDE (targetFQNHash, tagFQN, labelType, state)
|
||||
WHERE state = 1;
|
||||
|
||||
-- Only create if you need %contains% searches
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
-- GIN index for substring matches (LIKE '%foo%')
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS gin_tag_usage_targetfqn_trgm
|
||||
ON tag_usage USING GIN (targetFQNHash gin_trgm_ops)
|
||||
WHERE state = 1;
|
||||
|
||||
-- Optimize autovacuum for tag_usage (high update frequency)
|
||||
ALTER TABLE tag_usage SET (
|
||||
autovacuum_vacuum_scale_factor = 0.05, -- Vacuum at 5% dead rows (default 20%)
|
||||
autovacuum_analyze_scale_factor = 0.02, -- Analyze at 2% changed rows (default 10%)
|
||||
autovacuum_vacuum_threshold = 50, -- Minimum rows before vacuum
|
||||
autovacuum_analyze_threshold = 50, -- Minimum rows before analyze
|
||||
fillfactor = 90 -- Leave 10% free space for HOT updates
|
||||
);
|
||||
|
||||
-- Increase statistics target for frequently queried columns
|
||||
ALTER TABLE tag_usage ALTER COLUMN targetFQNHash SET STATISTICS 1000;
|
||||
ALTER TABLE tag_usage ALTER COLUMN targetfqnhash_lower SET STATISTICS 1000;
|
||||
ALTER TABLE tag_usage ALTER COLUMN tagFQN SET STATISTICS 500;
|
||||
ALTER TABLE tag_usage ALTER COLUMN tagfqn_lower SET STATISTICS 500;
|
||||
ALTER TABLE tag_usage ALTER COLUMN source SET STATISTICS 100;
|
||||
|
||||
-- Add index for efficient bulk term count queries
|
||||
-- The bulkGetTermCounts query uses: WHERE classificationHash IN (...) AND deleted = FALSE
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tag_classification_deleted
|
||||
ON tag (classificationHash, deleted);
|
||||
|
||||
-- Create new indexes with deleted column for efficient filtering
|
||||
-- Using partial indexes (WHERE deleted = FALSE) for even better performance
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_relationship_from_deleted
|
||||
ON entity_relationship(fromId, fromEntity, relation)
|
||||
INCLUDE (toId, toEntity, relation)
|
||||
WHERE deleted = FALSE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_relationship_to_deleted
|
||||
ON entity_relationship(toId, toEntity, relation)
|
||||
INCLUDE (fromId, fromEntity, relation)
|
||||
WHERE deleted = FALSE;
|
||||
|
||||
-- Also add indexes for the specific queries that include fromEntity/toEntity filters
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_relationship_from_typed
|
||||
ON entity_relationship(toId, toEntity, relation, fromEntity)
|
||||
INCLUDE (fromEntity, toEntity)
|
||||
WHERE deleted = FALSE;
|
||||
|
||||
-- Index for bidirectional lookups (used in UNION queries)
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_relationship_bidirectional
|
||||
ON entity_relationship(fromId, toId, relation)
|
||||
WHERE deleted = FALSE;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
UPDATE
|
||||
classification
|
||||
SET
|
||||
json = JSON_SET(
|
||||
json,
|
||||
'$.autoClassificationConfig',
|
||||
CAST(
|
||||
'{"enabled": true, "conflictResolution": "highest_priority", "minimumConfidence": 0.6, "requireExplicitMatch": true}'
|
||||
AS JSON
|
||||
)
|
||||
)
|
||||
WHERE
|
||||
JSON_VALUE(json, '$.name' RETURNING CHAR) = 'PII';
|
||||
@@ -0,0 +1,61 @@
|
||||
-- Add Flowable batch processing tables for history cleanup support
|
||||
CREATE TABLE IF NOT EXISTS flw_ru_batch (
|
||||
id_ VARCHAR(64) NOT NULL,
|
||||
rev_ INTEGER,
|
||||
type_ VARCHAR(64) NOT NULL,
|
||||
search_key_ VARCHAR(255),
|
||||
search_key2_ VARCHAR(255),
|
||||
create_time_ TIMESTAMP(3) NOT NULL,
|
||||
complete_time_ TIMESTAMP(3),
|
||||
status_ VARCHAR(255),
|
||||
batch_doc_id_ VARCHAR(64),
|
||||
tenant_id_ VARCHAR(255) DEFAULT '',
|
||||
CONSTRAINT pk_flw_ru_batch PRIMARY KEY (id_)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_flw_ru_batch_type ON flw_ru_batch (type_);
|
||||
CREATE INDEX idx_flw_ru_batch_search_key ON flw_ru_batch (search_key_);
|
||||
CREATE INDEX idx_flw_ru_batch_status ON flw_ru_batch (status_);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS flw_ru_batch_part (
|
||||
id_ VARCHAR(64) NOT NULL,
|
||||
rev_ INTEGER,
|
||||
batch_id_ VARCHAR(64),
|
||||
type_ VARCHAR(64) NOT NULL,
|
||||
scope_id_ VARCHAR(64),
|
||||
sub_scope_id_ VARCHAR(64),
|
||||
scope_type_ VARCHAR(64),
|
||||
search_key_ VARCHAR(255),
|
||||
search_key2_ VARCHAR(255),
|
||||
create_time_ TIMESTAMP(3) NOT NULL,
|
||||
complete_time_ TIMESTAMP(3),
|
||||
status_ VARCHAR(255),
|
||||
result_doc_id_ VARCHAR(64),
|
||||
tenant_id_ VARCHAR(255) DEFAULT '',
|
||||
CONSTRAINT pk_flw_ru_batch_part PRIMARY KEY (id_)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_flw_ru_batch_part_batch_id ON flw_ru_batch_part (batch_id_);
|
||||
CREATE INDEX idx_flw_ru_batch_part_type ON flw_ru_batch_part (type_);
|
||||
CREATE INDEX idx_flw_ru_batch_part_status ON flw_ru_batch_part (status_);
|
||||
|
||||
-- Update workflow settings with new history cleanup configuration fields
|
||||
UPDATE openmetadata_settings
|
||||
SET json = JSON_SET(
|
||||
JSON_SET(
|
||||
JSON_SET(
|
||||
JSON_SET(
|
||||
json,
|
||||
'$.historyCleanUpConfiguration.batchSize',
|
||||
1000
|
||||
),
|
||||
'$.historyCleanUpConfiguration.timeCycleConfig',
|
||||
'0 0 0 ? * 1'
|
||||
),
|
||||
'$.runTimeCleanUpConfiguration',
|
||||
JSON_OBJECT()
|
||||
),
|
||||
'$.runTimeCleanUpConfiguration.batchSize',
|
||||
500
|
||||
)
|
||||
WHERE configType = 'workflowSettings';
|
||||
@@ -0,0 +1,13 @@
|
||||
UPDATE
|
||||
classification
|
||||
SET
|
||||
json = json::jsonb || json_build_object(
|
||||
'autoClassificationConfig', json_build_object(
|
||||
'enabled', true,
|
||||
'conflictResolution', 'highest_priority',
|
||||
'minimumConfidence', 0.6,
|
||||
'requireExplicitMatch', true
|
||||
)
|
||||
)::jsonb
|
||||
WHERE
|
||||
json->>'name' = 'PII';
|
||||
@@ -0,0 +1,65 @@
|
||||
-- Add Flowable batch processing tables for history cleanup support
|
||||
CREATE TABLE IF NOT EXISTS flw_ru_batch (
|
||||
id_ VARCHAR(64) NOT NULL,
|
||||
rev_ INTEGER,
|
||||
type_ VARCHAR(64) NOT NULL,
|
||||
search_key_ VARCHAR(255),
|
||||
search_key2_ VARCHAR(255),
|
||||
create_time_ TIMESTAMP(3) NOT NULL,
|
||||
complete_time_ TIMESTAMP(3),
|
||||
status_ VARCHAR(255),
|
||||
batch_doc_id_ VARCHAR(64),
|
||||
tenant_id_ VARCHAR(255) DEFAULT '',
|
||||
CONSTRAINT pk_flw_ru_batch PRIMARY KEY (id_)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_flw_ru_batch_type ON flw_ru_batch (type_);
|
||||
CREATE INDEX idx_flw_ru_batch_search_key ON flw_ru_batch (search_key_);
|
||||
CREATE INDEX idx_flw_ru_batch_status ON flw_ru_batch (status_);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS flw_ru_batch_part (
|
||||
id_ VARCHAR(64) NOT NULL,
|
||||
rev_ INTEGER,
|
||||
batch_id_ VARCHAR(64),
|
||||
type_ VARCHAR(64) NOT NULL,
|
||||
scope_id_ VARCHAR(64),
|
||||
sub_scope_id_ VARCHAR(64),
|
||||
scope_type_ VARCHAR(64),
|
||||
search_key_ VARCHAR(255),
|
||||
search_key2_ VARCHAR(255),
|
||||
create_time_ TIMESTAMP(3) NOT NULL,
|
||||
complete_time_ TIMESTAMP(3),
|
||||
status_ VARCHAR(255),
|
||||
result_doc_id_ VARCHAR(64),
|
||||
tenant_id_ VARCHAR(255) DEFAULT '',
|
||||
CONSTRAINT pk_flw_ru_batch_part PRIMARY KEY (id_)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_flw_ru_batch_part_batch_id ON flw_ru_batch_part (batch_id_);
|
||||
CREATE INDEX idx_flw_ru_batch_part_type ON flw_ru_batch_part (type_);
|
||||
CREATE INDEX idx_flw_ru_batch_part_status ON flw_ru_batch_part (status_);
|
||||
|
||||
-- Update workflow settings with new history cleanup configuration fields
|
||||
UPDATE openmetadata_settings
|
||||
SET json = jsonb_set(
|
||||
jsonb_set(
|
||||
jsonb_set(
|
||||
jsonb_set(
|
||||
json,
|
||||
'{historyCleanUpConfiguration,batchSize}',
|
||||
'1000',
|
||||
true
|
||||
),
|
||||
'{historyCleanUpConfiguration,timeCycleConfig}',
|
||||
'"0 0 0 ? * 1"',
|
||||
true
|
||||
),
|
||||
'{runTimeCleanUpConfiguration}',
|
||||
'{}',
|
||||
true
|
||||
),
|
||||
'{runTimeCleanUpConfiguration,batchSize}',
|
||||
'500',
|
||||
true
|
||||
)
|
||||
WHERE configtype = 'workflowSettings';
|
||||
@@ -0,0 +1,13 @@
|
||||
-- Migrate REST API service connections: move flat openAPISchemaURL into nested openAPISchemaConnection object
|
||||
UPDATE api_service_entity
|
||||
SET json = JSON_SET(
|
||||
JSON_REMOVE(json, '$.connection.config.openAPISchemaURL'),
|
||||
'$.connection.config.openAPISchemaConnection',
|
||||
JSON_OBJECT(
|
||||
'openAPISchemaURL',
|
||||
JSON_UNQUOTE(JSON_EXTRACT(json, '$.connection.config.openAPISchemaURL'))
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'Rest'
|
||||
AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.openAPISchemaURL')
|
||||
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.openAPISchemaConnection');
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Migrate REST API service connections: move flat openAPISchemaURL into nested openAPISchemaConnection object
|
||||
UPDATE api_service_entity
|
||||
SET json = jsonb_set(
|
||||
json #- '{connection,config,openAPISchemaURL}',
|
||||
'{connection,config,openAPISchemaConnection}',
|
||||
jsonb_build_object('openAPISchemaURL', json #> '{connection,config,openAPISchemaURL}'),
|
||||
true
|
||||
)
|
||||
WHERE serviceType = 'Rest'
|
||||
AND jsonb_exists(json -> 'connection' -> 'config', 'openAPISchemaURL')
|
||||
AND NOT jsonb_exists(json -> 'connection' -> 'config', 'openAPISchemaConnection');
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Migrate OpenLineage connection config: move Kafka-specific fields into brokerConfig
|
||||
-- Supports the schema change adding Kinesis as an alternative broker type
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = JSON_SET(
|
||||
JSON_REMOVE(json,
|
||||
'$.connection.config.brokersUrl',
|
||||
'$.connection.config.topicName',
|
||||
'$.connection.config.consumerGroupName',
|
||||
'$.connection.config.consumerOffsets',
|
||||
'$.connection.config.poolTimeout',
|
||||
'$.connection.config.sessionTimeout',
|
||||
'$.connection.config.securityProtocol',
|
||||
'$.connection.config.sslConfig',
|
||||
'$.connection.config.saslConfig'
|
||||
),
|
||||
'$.connection.config.brokerConfig',
|
||||
JSON_REMOVE(
|
||||
JSON_EXTRACT(json, '$.connection.config'),
|
||||
'$.type',
|
||||
'$.pipelineFilterPattern',
|
||||
'$.supportsMetadataExtraction'
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'OpenLineage'
|
||||
AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokersUrl')
|
||||
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokerConfig');
|
||||
@@ -0,0 +1,13 @@
|
||||
-- Migrate Redshift connection: move password into authType.password (basicAuth)
|
||||
UPDATE dbservice_entity
|
||||
SET json = JSON_INSERT(
|
||||
JSON_REMOVE(json, '$.connection.config.password'),
|
||||
'$.connection.config.authType',
|
||||
JSON_OBJECT(
|
||||
'password',
|
||||
JSON_EXTRACT(json, '$.connection.config.password')
|
||||
)
|
||||
)
|
||||
WHERE serviceType = 'Redshift'
|
||||
AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL
|
||||
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.authType');
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Migrate OpenLineage connection config: move Kafka-specific fields into brokerConfig
|
||||
-- Supports the schema change adding Kinesis as an alternative broker type
|
||||
UPDATE pipeline_service_entity
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{connection,config,brokerConfig}',
|
||||
(json #> '{connection,config}') - 'type' - 'pipelineFilterPattern' - 'supportsMetadataExtraction',
|
||||
true
|
||||
)
|
||||
#- '{connection,config,brokersUrl}'
|
||||
#- '{connection,config,topicName}'
|
||||
#- '{connection,config,consumerGroupName}'
|
||||
#- '{connection,config,consumerOffsets}'
|
||||
#- '{connection,config,poolTimeout}'
|
||||
#- '{connection,config,sessionTimeout}'
|
||||
#- '{connection,config,securityProtocol}'
|
||||
#- '{connection,config,sslConfig}'
|
||||
#- '{connection,config,saslConfig}'
|
||||
WHERE serviceType = 'OpenLineage'
|
||||
AND jsonb_exists(json #> '{connection,config}', 'brokersUrl')
|
||||
AND NOT jsonb_exists(json #> '{connection,config}', 'brokerConfig');
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Migrate Redshift connection: move password into authType.password (basicAuth)
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json #- '{connection,config,password}',
|
||||
'{connection,config,authType}',
|
||||
jsonb_build_object('password', json #> '{connection,config,password}')
|
||||
)
|
||||
WHERE serviceType = 'Redshift'
|
||||
AND json #> '{connection,config,password}' IS NOT NULL
|
||||
AND NOT jsonb_exists(json #> '{connection,config}', 'authType');
|
||||
@@ -0,0 +1,3 @@
|
||||
-- No schema changes required for this migration
|
||||
-- This migration only fixes FQN hash for entities with service names containing dots
|
||||
-- The data migration is handled in Java code
|
||||
@@ -0,0 +1,3 @@
|
||||
-- No schema changes required for this migration
|
||||
-- This migration only fixes FQN hash for entities with service names containing dots
|
||||
-- The data migration is handled in Java code
|
||||
@@ -0,0 +1,48 @@
|
||||
-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValueLengthsToBeBetween'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesMissingCount'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToBeInSet'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToBeNotInSet'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToBeNotNull'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToBeUnique'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToMatchRegex'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
|
||||
WHERE name = 'columnValuesToNotMatchRegex'
|
||||
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
|
||||
@@ -0,0 +1,2 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1114
|
||||
@@ -0,0 +1,80 @@
|
||||
-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValueLengthsToBeBetween'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesMissingCount'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeInSet'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeNotInSet'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeNotNull'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToBeUnique'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToMatchRegex'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedDataTypes}',
|
||||
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
|
||||
)
|
||||
WHERE name = 'columnValuesToNotMatchRegex'
|
||||
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
|
||||
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
|
||||
@@ -0,0 +1,2 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1114
|
||||
@@ -0,0 +1,7 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1114
|
||||
|
||||
-- Include `tag_usage.appliedAt` and `tag_usage.appliedBy` fields
|
||||
|
||||
ALTER TABLE tag_usage ADD appliedAt TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP;
|
||||
ALTER TABLE tag_usage ADD appliedBy VARCHAR(64) NULL DEFAULT 'admin';
|
||||
@@ -0,0 +1,14 @@
|
||||
-- DO NOT RENAME OR DELETE THIS FILE
|
||||
-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1114
|
||||
|
||||
-- Include `tag_usage.appliedAt` field
|
||||
-- Add nullable column and then include default to avoid backfill
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN appliedAt TIMESTAMP;
|
||||
|
||||
ALTER TABLE tag_usage
|
||||
ALTER COLUMN appliedAt SET DEFAULT NOW();
|
||||
|
||||
-- Include `tag_usage.appliedBy` field
|
||||
ALTER TABLE tag_usage
|
||||
ADD COLUMN appliedBy VARCHAR(64) DEFAULT 'admin';
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Remove overrideLineage from DatabaseMetadata pipeline config
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json, '$.sourceConfig.config.overrideLineage')
|
||||
WHERE JSON_EXTRACT(json, '$.sourceConfig.config.type') = 'DatabaseMetadata';
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Remove overrideLineage from DatabaseMetadata pipeline config
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = json::jsonb #- '{sourceConfig,config,overrideLineage}'
|
||||
WHERE json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata';
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Migrate Salesforce connection from sobjectName (string) to sobjectNames (array)
|
||||
-- Converts sobjectName to sobjectNames array and removes the old field
|
||||
UPDATE dbservice_entity
|
||||
SET
|
||||
json = JSON_REMOVE (
|
||||
JSON_SET (
|
||||
json,
|
||||
'$.connection.config.sobjectNames',
|
||||
JSON_ARRAY (
|
||||
JSON_UNQUOTE (
|
||||
JSON_EXTRACT (json, '$.connection.config.sobjectName')
|
||||
)
|
||||
)
|
||||
),
|
||||
'$.connection.config.sobjectName'
|
||||
)
|
||||
WHERE
|
||||
serviceType = 'Salesforce'
|
||||
AND JSON_TYPE (
|
||||
JSON_EXTRACT (json, '$.connection.config.sobjectName')
|
||||
) != 'NULL';
|
||||
|
||||
-- Upgrade appliedAt to microsecond precision to match PostgreSQL behavior.
|
||||
-- Without this, MySQL returns second-precision timestamps which cause spurious
|
||||
-- diffs in JSON patch operations, leading to deserialization failures.
|
||||
ALTER TABLE tag_usage MODIFY appliedAt TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6);
|
||||
@@ -0,0 +1,13 @@
|
||||
-- Migrate Salesforce connection from sobjectName (string) to sobjectNames (array)
|
||||
-- Converts sobjectName to sobjectNames array and removes the old field
|
||||
|
||||
UPDATE dbservice_entity
|
||||
SET json = jsonb_set(
|
||||
json::jsonb #- '{connection,config,sobjectName}',
|
||||
'{connection,config,sobjectNames}',
|
||||
jsonb_build_array(json->'connection'->'config'->>'sobjectName')
|
||||
)::json
|
||||
WHERE serviceType = 'Salesforce'
|
||||
AND json->'connection'->'config'->>'sobjectName' IS NOT NULL;
|
||||
|
||||
-- No changes needed for PostgreSQL - TIMESTAMP already has microsecond precision.
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Change entity_extension_time_series.timestamp from VIRTUAL to STORED for performance.
|
||||
-- STORED columns are materialized on disk, making unique constraint checks and range
|
||||
-- queries on timestamp significantly faster (especially for bulk pipeline status upserts).
|
||||
-- MySQL does not allow ALTER from VIRTUAL to STORED directly, so we drop and re-add.
|
||||
-- NOTE: This will lock the table for a full rebuild. On large deployments with millions
|
||||
-- of rows in entity_extension_time_series, plan for downtime accordingly.
|
||||
ALTER TABLE entity_extension_time_series
|
||||
DROP INDEX entity_extension_time_series_constraint,
|
||||
DROP COLUMN `timestamp`,
|
||||
ADD COLUMN `timestamp` bigint unsigned GENERATED ALWAYS AS (json_unquote(json_extract(`json`, _utf8mb4'$.timestamp'))) STORED NOT NULL,
|
||||
ADD UNIQUE KEY `entity_extension_time_series_constraint` (`entityFQNHash`, `extension`, `timestamp`);
|
||||
@@ -0,0 +1,3 @@
|
||||
-- No changes needed for entity_extension_time_series.timestamp on PostgreSQL.
|
||||
-- PostgreSQL already uses STORED for the generated timestamp column (since table creation).
|
||||
-- MySQL migration changes it from VIRTUAL to STORED for consistency and performance.
|
||||
@@ -0,0 +1,78 @@
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.supportedServices',
|
||||
JSON_ARRAY('Snowflake', 'BigQuery', 'Athena', 'Redshift', 'Postgres', 'MySQL', 'Mssql', 'Oracle', 'Trino', 'SapHana')
|
||||
)
|
||||
WHERE name = 'tableDiff'
|
||||
AND (
|
||||
JSON_EXTRACT(json, '$.supportedServices') IS NULL
|
||||
OR JSON_LENGTH(JSON_EXTRACT(json, '$.supportedServices')) = 0
|
||||
);
|
||||
|
||||
-- Add timezone parameter to tableDataToBeFresh test definition
|
||||
UPDATE test_definition
|
||||
SET json = JSON_ARRAY_APPEND(
|
||||
json,
|
||||
'$.parameterDefinition',
|
||||
JSON_OBJECT(
|
||||
'name', 'timezone',
|
||||
'displayName', 'Timezone',
|
||||
'description', 'Timezone for the range calculation',
|
||||
'dataType', 'STRING',
|
||||
'required', false,
|
||||
'optionValues', JSON_ARRAY(
|
||||
'UTC',
|
||||
'Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Cairo',
|
||||
'Africa/Casablanca', 'Africa/Johannesburg', 'Africa/Lagos', 'Africa/Nairobi', 'Africa/Tunis',
|
||||
'America/Anchorage', 'America/Argentina/Buenos_Aires', 'America/Bogota', 'America/Caracas',
|
||||
'America/Chicago', 'America/Denver', 'America/Edmonton', 'America/Halifax', 'America/Havana',
|
||||
'America/Lima', 'America/Los_Angeles', 'America/Manaus', 'America/Mexico_City', 'America/New_York',
|
||||
'America/Panama', 'America/Phoenix', 'America/Puerto_Rico', 'America/Regina', 'America/Santiago',
|
||||
'America/Sao_Paulo', 'America/St_Johns', 'America/Toronto', 'America/Vancouver', 'America/Winnipeg',
|
||||
'Antarctica/McMurdo',
|
||||
'Asia/Almaty', 'Asia/Amman', 'Asia/Baghdad', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Beirut',
|
||||
'Asia/Colombo', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dubai', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong',
|
||||
'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Karachi', 'Asia/Kathmandu',
|
||||
'Asia/Kolkata', 'Asia/Kuala_Lumpur', 'Asia/Kuwait', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia',
|
||||
'Asia/Novosibirsk', 'Asia/Qatar', 'Asia/Riyadh', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore',
|
||||
'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tehran', 'Asia/Tokyo', 'Asia/Vladivostok', 'Asia/Yangon',
|
||||
'Asia/Yekaterinburg',
|
||||
'Atlantic/Azores', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Reykjavik',
|
||||
'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Darwin', 'Australia/Hobart',
|
||||
'Australia/Melbourne', 'Australia/Perth', 'Australia/Sydney',
|
||||
'Europe/Amsterdam', 'Europe/Athens', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Brussels',
|
||||
'Europe/Bucharest', 'Europe/Budapest', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Helsinki',
|
||||
'Europe/Kyiv', 'Europe/Lisbon', 'Europe/London', 'Europe/Madrid', 'Europe/Malta', 'Europe/Minsk',
|
||||
'Europe/Moscow', 'Europe/Oslo', 'Europe/Paris', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome',
|
||||
'Europe/Samara', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Vienna',
|
||||
'Europe/Vilnius', 'Europe/Warsaw', 'Europe/Zurich',
|
||||
'Indian/Maldives', 'Indian/Mauritius',
|
||||
'Pacific/Auckland', 'Pacific/Fiji', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Noumea',
|
||||
'Pacific/Pago_Pago', 'Pacific/Port_Moresby', 'Pacific/Tahiti', 'Pacific/Tongatapu'
|
||||
),
|
||||
'defaultValue', 'UTC'
|
||||
)
|
||||
)
|
||||
WHERE name = 'tableDataToBeFresh'
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.parameterDefinition[*].name'), '"timezone"');
|
||||
|
||||
-- Add validatorClass to all system test definitions (set to PascalCase name + 'Validator' if not present)
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(json, '$.validatorClass', CONCAT(UPPER(SUBSTRING(JSON_UNQUOTE(JSON_EXTRACT(json, '$.name')), 1, 1)), SUBSTRING(JSON_UNQUOTE(JSON_EXTRACT(json, '$.name')), 2), 'Validator'))
|
||||
WHERE JSON_EXTRACT(json, '$.validatorClass') IS NULL
|
||||
AND JSON_EXTRACT(json, '$.provider') = 'system';
|
||||
|
||||
-- Remove orphaned inputPorts and outputPorts fields from data_product_entity
|
||||
-- These fields were added in 1.10.x but removed in 1.12.x (now relationship-based)
|
||||
UPDATE data_product_entity
|
||||
SET json = JSON_REMOVE(JSON_REMOVE(json, '$.inputPorts'), '$.outputPorts')
|
||||
WHERE JSON_CONTAINS_PATH(json, 'one', '$.inputPorts')
|
||||
OR JSON_CONTAINS_PATH(json, 'one', '$.outputPorts');
|
||||
|
||||
-- Remove orphaned inputPorts and outputPorts fields from entity_extension (version history)
|
||||
UPDATE entity_extension
|
||||
SET json = JSON_REMOVE(JSON_REMOVE(json, '$.inputPorts'), '$.outputPorts')
|
||||
WHERE jsonSchema = 'dataProduct'
|
||||
AND (JSON_CONTAINS_PATH(json, 'one', '$.inputPorts')
|
||||
OR JSON_CONTAINS_PATH(json, 'one', '$.outputPorts'));
|
||||
@@ -0,0 +1,379 @@
|
||||
-- Update ApplicationBotRole to include Trigger operation
|
||||
UPDATE policy_entity
|
||||
SET json = JSON_ARRAY_APPEND(json, '$.rules[0].operations', 'Trigger')
|
||||
WHERE name = 'ApplicationBotPolicy'
|
||||
AND JSON_EXTRACT(json, '$.rules[0].operations') IS NOT NULL
|
||||
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.rules[0].operations'), '"Trigger"');
|
||||
|
||||
-- Create table for persisted audit log events
|
||||
CREATE TABLE IF NOT EXISTS audit_log_event (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT,
|
||||
change_event_id CHAR(36) NOT NULL,
|
||||
event_ts BIGINT NOT NULL,
|
||||
event_type VARCHAR(32) NOT NULL,
|
||||
user_name VARCHAR(256) DEFAULT NULL,
|
||||
actor_type VARCHAR(32) DEFAULT 'USER',
|
||||
impersonated_by VARCHAR(256) DEFAULT NULL,
|
||||
service_name VARCHAR(256) DEFAULT NULL,
|
||||
entity_type VARCHAR(128) DEFAULT NULL,
|
||||
entity_id CHAR(36) DEFAULT NULL,
|
||||
entity_fqn VARCHAR(768) DEFAULT NULL,
|
||||
entity_fqn_hash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
|
||||
event_json LONGTEXT NOT NULL,
|
||||
created_at BIGINT DEFAULT (UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY idx_audit_log_event_change_event_id (change_event_id),
|
||||
KEY idx_audit_log_event_ts (event_ts DESC),
|
||||
KEY idx_audit_log_event_user_ts (user_name, event_ts DESC),
|
||||
KEY idx_audit_log_event_entity_hash_ts (entity_fqn_hash, event_ts DESC),
|
||||
KEY idx_audit_log_actor_type_ts (actor_type, event_ts DESC),
|
||||
KEY idx_audit_log_service_name_ts (service_name, event_ts DESC),
|
||||
KEY idx_audit_log_created_at (created_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
|
||||
|
||||
-- Add enabled field to test_definition table for Rules Library feature
|
||||
-- This allows administrators to enable/disable test definitions in the rules library
|
||||
|
||||
-- Add virtual column for enabled field
|
||||
-- CAST is needed to convert JSON boolean (true/false) to TINYINT (1/0)
|
||||
ALTER TABLE test_definition
|
||||
ADD COLUMN enabled TINYINT(1)
|
||||
GENERATED ALWAYS AS (COALESCE(CAST(json_extract(json, '$.enabled') AS UNSIGNED), 1))
|
||||
VIRTUAL;
|
||||
|
||||
-- Add index for filtering enabled/disabled test definitions
|
||||
CREATE INDEX idx_test_definition_enabled ON test_definition(enabled);
|
||||
|
||||
-- Set all existing test definitions to enabled by default
|
||||
UPDATE test_definition
|
||||
SET json = JSON_SET(json, '$.enabled', true)
|
||||
WHERE json_extract(json, '$.enabled') IS NULL;
|
||||
|
||||
-- Migrate termsOfUse from string to object with content and inherited fields
|
||||
-- This converts existing termsOfUse string values to the new object structure: { "content": "...", "inherited": false }
|
||||
UPDATE data_contract_entity
|
||||
SET json = JSON_SET(
|
||||
json,
|
||||
'$.termsOfUse',
|
||||
JSON_OBJECT('content', JSON_UNQUOTE(JSON_EXTRACT(json, '$.termsOfUse')), 'inherited', false)
|
||||
)
|
||||
WHERE JSON_TYPE(JSON_EXTRACT(json, '$.termsOfUse')) = 'STRING';
|
||||
|
||||
-- Add updatedAt generated column to entity_extension table for efficient timestamp-based queries
|
||||
-- This supports the listEntityHistoryByTimestamp API endpoint for retrieving entity versions within a time range
|
||||
ALTER TABLE entity_extension
|
||||
ADD COLUMN updatedAt BIGINT UNSIGNED
|
||||
GENERATED ALWAYS AS (CAST(json_unquote(json_extract(json, '$.updatedAt')) AS UNSIGNED))
|
||||
STORED;
|
||||
|
||||
-- Create composite index for timestamp-based queries with cursor pagination
|
||||
-- This index supports queries that filter by updatedAt range and order by (updatedAt DESC, id DESC)
|
||||
CREATE INDEX idx_entity_extension_updated_at_id ON entity_extension(updatedAt DESC, id DESC);
|
||||
|
||||
-- Add composite indexes on entity tables for timestamp-based history queries with cursor pagination
|
||||
CREATE INDEX idx_table_entity_updated_at_id ON table_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_database_entity_updated_at_id ON database_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_database_schema_entity_updated_at_id ON database_schema_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_dashboard_entity_updated_at_id ON dashboard_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_pipeline_entity_updated_at_id ON pipeline_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_topic_entity_updated_at_id ON topic_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_chart_entity_updated_at_id ON chart_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_ml_model_entity_updated_at_id ON ml_model_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_stored_procedure_entity_updated_at_id ON stored_procedure_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_dashboard_data_model_entity_updated_at_id ON dashboard_data_model_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_storage_container_entity_updated_at_id ON storage_container_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_search_index_entity_updated_at_id ON search_index_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_glossary_entity_updated_at_id ON glossary_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_glossary_term_entity_updated_at_id ON glossary_term_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_tag_updated_at_id ON tag(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_classification_updated_at_id ON classification(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_data_product_entity_updated_at_id ON data_product_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_domain_entity_updated_at_id ON domain_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_user_entity_updated_at_id ON user_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_team_entity_updated_at_id ON team_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_dbservice_entity_updated_at_id ON dbservice_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_messaging_service_entity_updated_at_id ON messaging_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_dashboard_service_entity_updated_at_id ON dashboard_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_pipeline_service_entity_updated_at_id ON pipeline_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_storage_service_entity_updated_at_id ON storage_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_mlmodel_service_entity_updated_at_id ON mlmodel_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_metadata_service_entity_updated_at_id ON metadata_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_search_service_entity_updated_at_id ON search_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_api_service_entity_updated_at_id ON api_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_ingestion_pipeline_entity_updated_at_id ON ingestion_pipeline_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_test_suite_updated_at_id ON test_suite(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_test_case_updated_at_id ON test_case(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_api_collection_entity_updated_at_id ON api_collection_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX idx_api_endpoint_entity_updated_at_id ON api_endpoint_entity(updatedAt DESC, id DESC);
|
||||
|
||||
-- Add metadata column to tag_usage table
|
||||
ALTER TABLE tag_usage ADD metadata JSON NULL;
|
||||
|
||||
-- Upgrade appliedAt to microsecond precision to match PostgreSQL behavior.
|
||||
-- Without this, MySQL returns second-precision timestamps which cause spurious
|
||||
-- diffs in JSON patch operations, leading to deserialization failures.
|
||||
ALTER TABLE tag_usage MODIFY appliedAt TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6);
|
||||
|
||||
-- Distributed Search Indexing Tables
|
||||
|
||||
-- Table to track reindex jobs across distributed servers
|
||||
CREATE TABLE IF NOT EXISTS search_index_job (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
jobConfiguration JSON NOT NULL,
|
||||
targetIndexPrefix VARCHAR(255),
|
||||
stagedIndexMapping JSON,
|
||||
totalRecords BIGINT NOT NULL DEFAULT 0,
|
||||
processedRecords BIGINT NOT NULL DEFAULT 0,
|
||||
successRecords BIGINT NOT NULL DEFAULT 0,
|
||||
failedRecords BIGINT NOT NULL DEFAULT 0,
|
||||
stats JSON,
|
||||
createdBy VARCHAR(256) NOT NULL,
|
||||
createdAt BIGINT NOT NULL,
|
||||
startedAt BIGINT,
|
||||
completedAt BIGINT,
|
||||
updatedAt BIGINT NOT NULL,
|
||||
errorMessage TEXT,
|
||||
-- Legacy fields (no longer used but kept for compatibility)
|
||||
registrationDeadline BIGINT,
|
||||
registeredServerCount INT,
|
||||
PRIMARY KEY (id),
|
||||
INDEX idx_search_index_job_status (status),
|
||||
INDEX idx_search_index_job_created (createdAt DESC)
|
||||
);
|
||||
|
||||
-- Table to track partitions within a reindex job
|
||||
CREATE TABLE IF NOT EXISTS search_index_partition (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
entityType VARCHAR(128) NOT NULL,
|
||||
partitionIndex INT NOT NULL,
|
||||
rangeStart BIGINT NOT NULL,
|
||||
rangeEnd BIGINT NOT NULL,
|
||||
estimatedCount BIGINT NOT NULL,
|
||||
workUnits BIGINT NOT NULL,
|
||||
priority INT NOT NULL DEFAULT 50,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
processingCursor BIGINT NOT NULL DEFAULT 0,
|
||||
processedCount BIGINT NOT NULL DEFAULT 0,
|
||||
successCount BIGINT NOT NULL DEFAULT 0,
|
||||
failedCount BIGINT NOT NULL DEFAULT 0,
|
||||
assignedServer VARCHAR(255),
|
||||
claimedAt BIGINT,
|
||||
startedAt BIGINT,
|
||||
completedAt BIGINT,
|
||||
lastUpdateAt BIGINT,
|
||||
lastError TEXT,
|
||||
retryCount INT NOT NULL DEFAULT 0,
|
||||
claimableAt BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uk_partition_job_entity_idx (jobId, entityType, partitionIndex),
|
||||
INDEX idx_partition_job (jobId),
|
||||
INDEX idx_partition_status_priority (status, priority DESC),
|
||||
INDEX idx_partition_claimed (claimedAt),
|
||||
INDEX idx_partition_assigned_server (jobId, assignedServer),
|
||||
INDEX idx_partition_claimable (jobId, status, claimableAt),
|
||||
CONSTRAINT fk_partition_job FOREIGN KEY (jobId) REFERENCES search_index_job(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Table for distributed lock to ensure only one reindex job runs at a time
|
||||
CREATE TABLE IF NOT EXISTS search_reindex_lock (
|
||||
lockKey VARCHAR(64) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(255) NOT NULL,
|
||||
acquiredAt BIGINT NOT NULL,
|
||||
lastHeartbeat BIGINT NOT NULL,
|
||||
expiresAt BIGINT NOT NULL,
|
||||
PRIMARY KEY (lockKey)
|
||||
);
|
||||
|
||||
-- Search Index Failures Table
|
||||
-- Purpose: Store individual failure records for entities that fail during reindexing
|
||||
|
||||
CREATE TABLE IF NOT EXISTS search_index_failures (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(256) NOT NULL,
|
||||
entityType VARCHAR(256) NOT NULL,
|
||||
entityId VARCHAR(36),
|
||||
entityFqn VARCHAR(1024),
|
||||
failureStage VARCHAR(32) NOT NULL,
|
||||
errorMessage LONGTEXT,
|
||||
stackTrace LONGTEXT,
|
||||
timestamp BIGINT NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
INDEX idx_search_index_failures_job_id (jobId),
|
||||
INDEX idx_search_index_failures_server_id (serverId),
|
||||
INDEX idx_search_index_failures_entity_type (entityType),
|
||||
INDEX idx_search_index_failures_timestamp (timestamp)
|
||||
);
|
||||
|
||||
-- Search Index Server Stats Table
|
||||
-- Purpose: Track per-server stats in distributed indexing mode
|
||||
|
||||
CREATE TABLE IF NOT EXISTS search_index_server_stats (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(256) NOT NULL,
|
||||
readerSuccess BIGINT DEFAULT 0,
|
||||
readerFailed BIGINT DEFAULT 0,
|
||||
readerWarnings BIGINT DEFAULT 0,
|
||||
sinkTotal BIGINT DEFAULT 0,
|
||||
sinkSuccess BIGINT DEFAULT 0,
|
||||
sinkFailed BIGINT DEFAULT 0,
|
||||
sinkWarnings BIGINT DEFAULT 0,
|
||||
entityBuildFailures BIGINT DEFAULT 0,
|
||||
partitionsCompleted INT DEFAULT 0,
|
||||
partitionsFailed INT DEFAULT 0,
|
||||
lastUpdatedAt BIGINT NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE INDEX idx_search_index_server_stats_job_server (jobId, serverId),
|
||||
INDEX idx_search_index_server_stats_job_id (jobId)
|
||||
);
|
||||
|
||||
-- Create Learning Resource Entity Table
|
||||
CREATE TABLE IF NOT EXISTS learning_resource_entity (
|
||||
id varchar(36) GENERATED ALWAYS AS (json_unquote(json_extract(`json`,'$.id'))) STORED NOT NULL,
|
||||
name varchar(3072) GENERATED ALWAYS AS (json_unquote(json_extract(`json`,'$.fullyQualifiedName'))) VIRTUAL,
|
||||
fqnHash varchar(256) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
|
||||
json json NOT NULL,
|
||||
updatedAt bigint UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(`json`,'$.updatedAt'))) VIRTUAL NOT NULL,
|
||||
updatedBy varchar(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`,'$.updatedBy'))) VIRTUAL NOT NULL,
|
||||
deleted TINYINT(1) GENERATED ALWAYS AS (IF(json_extract(json,'$.deleted') = TRUE, 1, 0)) VIRTUAL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY fqnHash (fqnHash)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
-- DELETE old workflow instances where status is null
|
||||
DELETE FROM workflow_instance_time_series
|
||||
WHERE JSON_EXTRACT(json, '$.status') IS NULL;
|
||||
|
||||
-- DELETE old workflow instance state where status is null
|
||||
DELETE FROM workflow_instance_state_time_series
|
||||
WHERE JSON_EXTRACT(json, '$.status') IS NULL;
|
||||
|
||||
-- Widen entityLink generated column from VARCHAR(255) to TEXT
|
||||
-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
|
||||
ALTER TABLE workflow_instance_time_series
|
||||
MODIFY COLUMN entityLink TEXT GENERATED ALWAYS AS (json ->> '$.variables.global_relatedEntity');
|
||||
|
||||
-- Add process and vector stage columns to search_index_server_stats table
|
||||
-- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
|
||||
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN processSuccess BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN processFailed BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN vectorSuccess BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN vectorFailed BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN vectorWarnings BIGINT DEFAULT 0;
|
||||
|
||||
-- Add entityType column to support per-entity stats tracking
|
||||
-- Stats are now tracked per (jobId, serverId, entityType) instead of (jobId, serverId)
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN entityType VARCHAR(128) NOT NULL DEFAULT 'unknown';
|
||||
|
||||
-- Drop old unique index and create new one with entityType
|
||||
ALTER TABLE search_index_server_stats DROP INDEX idx_search_index_server_stats_job_server;
|
||||
CREATE UNIQUE INDEX idx_search_index_server_stats_job_server_entity
|
||||
ON search_index_server_stats (jobId, serverId, entityType);
|
||||
|
||||
-- Remove deprecated columns (entityBuildFailures is redundant - failures are tracked as processFailed)
|
||||
-- sinkTotal and sinkWarnings are not needed
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN entityBuildFailures;
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN sinkTotal;
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN sinkWarnings;
|
||||
|
||||
-- Create ai_application_entity table
|
||||
CREATE TABLE IF NOT EXISTS ai_application_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_name (fqnHash),
|
||||
INDEX name_index (name),
|
||||
INDEX deleted_index (deleted)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Application entities';
|
||||
|
||||
-- Create llm_model_entity table
|
||||
CREATE TABLE IF NOT EXISTS llm_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_name (fqnHash),
|
||||
INDEX name_index (name),
|
||||
INDEX deleted_index (deleted)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Model entities';
|
||||
|
||||
-- Create prompt_template_entity table
|
||||
CREATE TABLE IF NOT EXISTS prompt_template_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_name (fqnHash),
|
||||
INDEX name_index (name),
|
||||
INDEX deleted_index (deleted)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Prompt Template entities';
|
||||
|
||||
-- Create agent_execution_entity table
|
||||
CREATE TABLE IF NOT EXISTS agent_execution_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
agentId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.agentId') STORED NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
INDEX agent_index (agentId),
|
||||
INDEX timestamp_index (timestamp)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Agent Execution logs';
|
||||
|
||||
-- Create ai_governance_policy_entity table
|
||||
CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_name (fqnHash),
|
||||
INDEX name_index (name),
|
||||
INDEX deleted_index (deleted)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Governance Policy entities';
|
||||
|
||||
-- Create llm_service_entity table
|
||||
CREATE TABLE IF NOT EXISTS llm_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.id'))) STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.name'))) VIRTUAL NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.serviceType'))) VIRTUAL NOT NULL,
|
||||
json JSON NOT NULL,
|
||||
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedAt'))) VIRTUAL NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedBy'))) VIRTUAL NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
|
||||
deleted TINYINT(1) GENERATED ALWAYS AS (json_extract(`json`, '$.deleted')) VIRTUAL,
|
||||
nameHash VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY nameHash (nameHash),
|
||||
INDEX name_index (name),
|
||||
INDEX service_type_index (serviceType),
|
||||
INDEX deleted_index (deleted)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Service entities';
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{supportedServices}',
|
||||
'["Snowflake", "BigQuery", "Athena", "Redshift", "Postgres", "MySQL", "Mssql", "Oracle", "Trino", "SapHana"]'::jsonb
|
||||
)
|
||||
WHERE name = 'tableDiff'
|
||||
AND (
|
||||
json->'supportedServices' IS NULL
|
||||
OR json->'supportedServices' = '[]'::jsonb
|
||||
);
|
||||
|
||||
-- Add timezone parameter to tableDataToBeFresh test definition
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{parameterDefinition}',
|
||||
(json->'parameterDefinition')::jsonb || jsonb_build_object(
|
||||
'name', 'timezone',
|
||||
'displayName', 'Timezone',
|
||||
'description', 'Timezone for the range calculation',
|
||||
'dataType', 'STRING',
|
||||
'required', false,
|
||||
'optionValues', '["UTC","Africa/Abidjan","Africa/Accra","Africa/Addis_Ababa","Africa/Algiers","Africa/Cairo","Africa/Casablanca","Africa/Johannesburg","Africa/Lagos","Africa/Nairobi","Africa/Tunis","America/Anchorage","America/Argentina/Buenos_Aires","America/Bogota","America/Caracas","America/Chicago","America/Denver","America/Edmonton","America/Halifax","America/Havana","America/Lima","America/Los_Angeles","America/Manaus","America/Mexico_City","America/New_York","America/Panama","America/Phoenix","America/Puerto_Rico","America/Regina","America/Santiago","America/Sao_Paulo","America/St_Johns","America/Toronto","America/Vancouver","America/Winnipeg","Antarctica/McMurdo","Asia/Almaty","Asia/Amman","Asia/Baghdad","Asia/Baku","Asia/Bangkok","Asia/Beirut","Asia/Colombo","Asia/Damascus","Asia/Dhaka","Asia/Dubai","Asia/Ho_Chi_Minh","Asia/Hong_Kong","Asia/Istanbul","Asia/Jakarta","Asia/Jerusalem","Asia/Kabul","Asia/Karachi","Asia/Kathmandu","Asia/Kolkata","Asia/Kuala_Lumpur","Asia/Kuwait","Asia/Manila","Asia/Muscat","Asia/Nicosia","Asia/Novosibirsk","Asia/Qatar","Asia/Riyadh","Asia/Seoul","Asia/Shanghai","Asia/Singapore","Asia/Taipei","Asia/Tashkent","Asia/Tehran","Asia/Tokyo","Asia/Vladivostok","Asia/Yangon","Asia/Yekaterinburg","Atlantic/Azores","Atlantic/Canary","Atlantic/Cape_Verde","Atlantic/Reykjavik","Australia/Adelaide","Australia/Brisbane","Australia/Darwin","Australia/Hobart","Australia/Melbourne","Australia/Perth","Australia/Sydney","Europe/Amsterdam","Europe/Athens","Europe/Belgrade","Europe/Berlin","Europe/Brussels","Europe/Bucharest","Europe/Budapest","Europe/Copenhagen","Europe/Dublin","Europe/Helsinki","Europe/Kyiv","Europe/Lisbon","Europe/London","Europe/Madrid","Europe/Malta","Europe/Minsk","Europe/Moscow","Europe/Oslo","Europe/Paris","Europe/Prague","Europe/Riga","Europe/Rome","Europe/Samara","Europe/Sofia","Europe/Stockholm","Europe/Tallinn","Europe/Vienna","Europe/Vilnius","Europe/Warsaw","Europe/Zurich","Indian/Maldives","Indian/Mauritius","Pacific/Auckland","Pacific/Fiji","Pacific/Guam","Pacific/Honolulu","Pacific/Noumea","Pacific/Pago_Pago","Pacific/Port_Moresby","Pacific/Tahiti","Pacific/Tongatapu"]'::jsonb,
|
||||
'defaultValue', 'UTC'
|
||||
)::jsonb
|
||||
)
|
||||
WHERE name = 'tableDataToBeFresh'
|
||||
AND NOT (json->'parameterDefinition' @> '[{"name": "timezone"}]'::jsonb);
|
||||
|
||||
-- Add parameters back in tableRowInsertedCountToBeBetween if missing
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(
|
||||
json,
|
||||
'{parameterDefinition}',
|
||||
(
|
||||
SELECT jsonb_agg(elem)
|
||||
FROM (
|
||||
-- Keep existing elements
|
||||
SELECT elem
|
||||
FROM jsonb_array_elements(json -> 'parameterDefinition') AS elem
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Add columnName if not exists
|
||||
SELECT jsonb_build_object(
|
||||
'name', 'columnName',
|
||||
'dataType', 'STRING',
|
||||
'required', true,
|
||||
'description', 'Name of the Column to check for new rows inserted.',
|
||||
'displayName', 'Column Name',
|
||||
'optionValues', '[]'::jsonb
|
||||
)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(json -> 'parameterDefinition') AS e
|
||||
WHERE e ->> 'name' = 'columnName'
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Add rangeType if not exists
|
||||
SELECT jsonb_build_object(
|
||||
'name', 'rangeType',
|
||||
'dataType', 'STRING',
|
||||
'required', true,
|
||||
'description', 'One of ''HOUR'', ''DAY'', ''MONTH'', ''YEAR'' to specify the range type for checking new rows inserted.',
|
||||
'displayName', 'Range Type',
|
||||
'optionValues', '[]'::jsonb
|
||||
)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(json -> 'parameterDefinition') AS e
|
||||
WHERE e ->> 'name' = 'rangeType'
|
||||
)
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Add rangeInterval if not exists
|
||||
SELECT jsonb_build_object(
|
||||
'name', 'rangeInterval',
|
||||
'dataType', 'INT',
|
||||
'required', true,
|
||||
'description', 'Interval Range. E.g. if rangeInterval=1 and rangeType=DAY, we''ll check the numbers of rows inserted where columnName=-1 DAY',
|
||||
'displayName', 'Interval',
|
||||
'optionValues', '[]'::jsonb
|
||||
)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(json -> 'parameterDefinition') AS e
|
||||
WHERE e ->> 'name' = 'rangeInterval'
|
||||
)
|
||||
) AS combined
|
||||
)
|
||||
)
|
||||
WHERE name = 'tableRowInsertedCountToBeBetween';
|
||||
|
||||
-- Add validatorClass to all system test definitions (set to PascalCase name + 'Validator' if not present)
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(json::jsonb, '{validatorClass}', to_jsonb(INITCAP(SUBSTRING(json->>'name', 1, 1)) || SUBSTRING(json->>'name', 2) || 'Validator'))
|
||||
WHERE json->>'validatorClass' IS NULL
|
||||
AND json->>'provider' = 'system';
|
||||
|
||||
-- Remove orphaned inputPorts and outputPorts fields from data_product_entity
|
||||
-- These fields were added in 1.10.x but removed in 1.12.x (now relationship-based)
|
||||
UPDATE data_product_entity
|
||||
SET json = json - 'inputPorts' - 'outputPorts'
|
||||
WHERE json ?? 'inputPorts' OR json ?? 'outputPorts';
|
||||
|
||||
-- Remove orphaned inputPorts and outputPorts fields from entity_extension (version history)
|
||||
UPDATE entity_extension
|
||||
SET json = json::jsonb - 'inputPorts' - 'outputPorts'
|
||||
WHERE jsonSchema = 'dataProduct'
|
||||
AND (json::jsonb ?? 'inputPorts' OR json::jsonb ?? 'outputPorts');
|
||||
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
-- Update ApplicationBotRole to include Trigger operation
|
||||
UPDATE policy_entity
|
||||
SET json = jsonb_set(json::jsonb, '{rules,0,operations}', (json->'rules'->0->'operations')::jsonb || '["Trigger"]'::jsonb)
|
||||
WHERE name = 'ApplicationBotPolicy'
|
||||
AND json->'rules'->0->'operations' IS NOT NULL
|
||||
AND NOT (json->'rules'->0->'operations' @> '"Trigger"'::jsonb);
|
||||
|
||||
-- Create table for persisted audit log events
|
||||
CREATE TABLE IF NOT EXISTS audit_log_event (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
change_event_id UUID NOT NULL,
|
||||
event_ts BIGINT NOT NULL,
|
||||
event_type VARCHAR(32) NOT NULL,
|
||||
user_name VARCHAR(256),
|
||||
actor_type VARCHAR(32) DEFAULT 'USER',
|
||||
impersonated_by VARCHAR(256) DEFAULT NULL,
|
||||
service_name VARCHAR(256) DEFAULT NULL,
|
||||
entity_type VARCHAR(128),
|
||||
entity_id UUID,
|
||||
entity_fqn VARCHAR(768),
|
||||
entity_fqn_hash VARCHAR(768),
|
||||
event_json TEXT NOT NULL,
|
||||
created_at BIGINT DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
|
||||
);
|
||||
|
||||
-- Add indexes for efficient filtering
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_actor_type_ts ON audit_log_event (actor_type, event_ts DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_service_name_ts ON audit_log_event (service_name, event_ts DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_created_at ON audit_log_event (created_at);
|
||||
|
||||
|
||||
-- Add enabled field to test_definition table for Rules Library feature
|
||||
-- This allows administrators to enable/disable test definitions in the rules library
|
||||
|
||||
-- Add generated column for enabled field with default true for existing rows
|
||||
ALTER TABLE test_definition
|
||||
ADD COLUMN IF NOT EXISTS enabled BOOLEAN GENERATED ALWAYS AS (
|
||||
COALESCE((json ->> 'enabled')::boolean, true)
|
||||
) STORED;
|
||||
|
||||
-- Add index for filtering enabled/disabled test definitions
|
||||
CREATE INDEX IF NOT EXISTS idx_test_definition_enabled ON test_definition(enabled);
|
||||
|
||||
-- Set all existing test definitions to enabled by default
|
||||
UPDATE test_definition
|
||||
SET json = jsonb_set(json::jsonb, '{enabled}', 'true'::jsonb, true)::json
|
||||
WHERE json ->> 'enabled' IS NULL;
|
||||
|
||||
-- Migrate termsOfUse from string to object with content and inherited fields
|
||||
-- This converts existing termsOfUse string values to the new object structure: { "content": "...", "inherited": false }
|
||||
UPDATE data_contract_entity
|
||||
SET json = jsonb_set(
|
||||
json::jsonb,
|
||||
'{termsOfUse}',
|
||||
jsonb_build_object('content', json ->> 'termsOfUse', 'inherited', false)
|
||||
)::json
|
||||
WHERE jsonb_typeof((json::jsonb) -> 'termsOfUse') = 'string';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_log_event_change_event_id
|
||||
ON audit_log_event (change_event_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_event_ts
|
||||
ON audit_log_event (event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_event_user_ts
|
||||
ON audit_log_event (user_name, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_event_entity_hash_ts
|
||||
ON audit_log_event (entity_fqn_hash, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_actor_type_ts
|
||||
ON audit_log_event (actor_type, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_service_name_ts
|
||||
ON audit_log_event (service_name, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_created_at
|
||||
ON audit_log_event (created_at);
|
||||
|
||||
-- Add updatedAt generated column to entity_extension table for efficient timestamp-based queries
|
||||
-- This supports the listEntityHistoryByTimestamp API endpoint for retrieving entity versions within a time range
|
||||
ALTER TABLE entity_extension
|
||||
ADD COLUMN IF NOT EXISTS updatedAt BIGINT GENERATED ALWAYS AS (
|
||||
(json ->> 'updatedAt')::BIGINT
|
||||
) STORED;
|
||||
|
||||
-- Create composite index for timestamp-based queries with cursor pagination
|
||||
-- This index supports queries that filter by updatedAt range and order by (updatedAt DESC, id DESC)
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_extension_updated_at_id ON entity_extension(updatedAt DESC, id DESC);
|
||||
|
||||
-- Add composite indexes on entity tables for timestamp-based history queries with cursor pagination
|
||||
CREATE INDEX IF NOT EXISTS idx_table_entity_updated_at_id ON table_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_database_entity_updated_at_id ON database_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_database_schema_entity_updated_at_id ON database_schema_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dashboard_entity_updated_at_id ON dashboard_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_pipeline_entity_updated_at_id ON pipeline_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_topic_entity_updated_at_id ON topic_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_chart_entity_updated_at_id ON chart_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ml_model_entity_updated_at_id ON ml_model_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_stored_procedure_entity_updated_at_id ON stored_procedure_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dashboard_data_model_entity_updated_at_id ON dashboard_data_model_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_storage_container_entity_updated_at_id ON storage_container_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_entity_updated_at_id ON search_index_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_glossary_entity_updated_at_id ON glossary_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_glossary_term_entity_updated_at_id ON glossary_term_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_updated_at_id ON tag(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_classification_updated_at_id ON classification(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_product_entity_updated_at_id ON data_product_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_domain_entity_updated_at_id ON domain_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_entity_updated_at_id ON user_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_team_entity_updated_at_id ON team_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dbservice_entity_updated_at_id ON dbservice_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_messaging_service_entity_updated_at_id ON messaging_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dashboard_service_entity_updated_at_id ON dashboard_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_pipeline_service_entity_updated_at_id ON pipeline_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_storage_service_entity_updated_at_id ON storage_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_mlmodel_service_entity_updated_at_id ON mlmodel_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_metadata_service_entity_updated_at_id ON metadata_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_service_entity_updated_at_id ON search_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_api_service_entity_updated_at_id ON api_service_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingestion_pipeline_entity_updated_at_id ON ingestion_pipeline_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_test_suite_updated_at_id ON test_suite(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_test_case_updated_at_id ON test_case(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_api_collection_entity_updated_at_id ON api_collection_entity(updatedAt DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_api_endpoint_entity_updated_at_id ON api_endpoint_entity(updatedAt DESC, id DESC);
|
||||
|
||||
|
||||
-- Add metadata column to tag_usage table
|
||||
ALTER TABLE tag_usage ADD COLUMN IF NOT EXISTS metadata JSON;
|
||||
|
||||
-- Distributed Search Indexing Tables
|
||||
|
||||
-- Table to track reindex jobs across distributed servers
|
||||
CREATE TABLE IF NOT EXISTS search_index_job (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
jobConfiguration JSONB NOT NULL,
|
||||
targetIndexPrefix VARCHAR(255),
|
||||
stagedIndexMapping JSONB,
|
||||
totalRecords BIGINT NOT NULL DEFAULT 0,
|
||||
processedRecords BIGINT NOT NULL DEFAULT 0,
|
||||
successRecords BIGINT NOT NULL DEFAULT 0,
|
||||
failedRecords BIGINT NOT NULL DEFAULT 0,
|
||||
stats JSONB,
|
||||
createdBy VARCHAR(256) NOT NULL,
|
||||
createdAt BIGINT NOT NULL,
|
||||
startedAt BIGINT,
|
||||
completedAt BIGINT,
|
||||
updatedAt BIGINT NOT NULL,
|
||||
errorMessage TEXT,
|
||||
-- Legacy fields (no longer used but kept for compatibility)
|
||||
registrationDeadline BIGINT,
|
||||
registeredServerCount INT,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_job_status ON search_index_job(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_job_created ON search_index_job(createdAt DESC);
|
||||
|
||||
-- Table to track partitions within a reindex job
|
||||
CREATE TABLE IF NOT EXISTS search_index_partition (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
entityType VARCHAR(128) NOT NULL,
|
||||
partitionIndex INT NOT NULL,
|
||||
rangeStart BIGINT NOT NULL,
|
||||
rangeEnd BIGINT NOT NULL,
|
||||
estimatedCount BIGINT NOT NULL,
|
||||
workUnits BIGINT NOT NULL,
|
||||
priority INT NOT NULL DEFAULT 50,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
processingCursor BIGINT NOT NULL DEFAULT 0,
|
||||
processedCount BIGINT NOT NULL DEFAULT 0,
|
||||
successCount BIGINT NOT NULL DEFAULT 0,
|
||||
failedCount BIGINT NOT NULL DEFAULT 0,
|
||||
assignedServer VARCHAR(255),
|
||||
claimedAt BIGINT,
|
||||
startedAt BIGINT,
|
||||
completedAt BIGINT,
|
||||
lastUpdateAt BIGINT,
|
||||
lastError TEXT,
|
||||
retryCount INT NOT NULL DEFAULT 0,
|
||||
claimableAt BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (jobId, entityType, partitionIndex),
|
||||
CONSTRAINT fk_partition_job FOREIGN KEY (jobId) REFERENCES search_index_job(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_partition_job ON search_index_partition(jobId);
|
||||
CREATE INDEX IF NOT EXISTS idx_partition_status_priority ON search_index_partition(status, priority DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_partition_claimed ON search_index_partition(claimedAt);
|
||||
CREATE INDEX IF NOT EXISTS idx_partition_assigned_server ON search_index_partition(jobId, assignedServer);
|
||||
CREATE INDEX IF NOT EXISTS idx_partition_claimable ON search_index_partition(jobId, status, claimableAt);
|
||||
|
||||
-- Table for distributed lock to ensure only one reindex job runs at a time
|
||||
CREATE TABLE IF NOT EXISTS search_reindex_lock (
|
||||
lockKey VARCHAR(64) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(255) NOT NULL,
|
||||
acquiredAt BIGINT NOT NULL,
|
||||
lastHeartbeat BIGINT NOT NULL,
|
||||
expiresAt BIGINT NOT NULL,
|
||||
PRIMARY KEY (lockKey)
|
||||
);
|
||||
|
||||
-- Search Index Failures Table
|
||||
-- Purpose: Store individual failure records for entities that fail during reindexing
|
||||
|
||||
CREATE TABLE IF NOT EXISTS search_index_failures (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(256) NOT NULL,
|
||||
entityType VARCHAR(256) NOT NULL,
|
||||
entityId VARCHAR(36),
|
||||
entityFqn VARCHAR(1024),
|
||||
failureStage VARCHAR(32) NOT NULL,
|
||||
errorMessage TEXT,
|
||||
stackTrace TEXT,
|
||||
timestamp BIGINT NOT NULL,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_failures_job_id ON search_index_failures(jobId);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_failures_server_id ON search_index_failures(serverId);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_failures_entity_type ON search_index_failures(entityType);
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_failures_timestamp ON search_index_failures(timestamp);
|
||||
|
||||
-- Search Index Server Stats Table
|
||||
-- Purpose: Track per-server stats in distributed indexing mode
|
||||
|
||||
CREATE TABLE IF NOT EXISTS search_index_server_stats (
|
||||
id VARCHAR(36) NOT NULL,
|
||||
jobId VARCHAR(36) NOT NULL,
|
||||
serverId VARCHAR(256) NOT NULL,
|
||||
readerSuccess BIGINT DEFAULT 0,
|
||||
readerFailed BIGINT DEFAULT 0,
|
||||
readerWarnings BIGINT DEFAULT 0,
|
||||
sinkTotal BIGINT DEFAULT 0,
|
||||
sinkSuccess BIGINT DEFAULT 0,
|
||||
sinkFailed BIGINT DEFAULT 0,
|
||||
sinkWarnings BIGINT DEFAULT 0,
|
||||
entityBuildFailures BIGINT DEFAULT 0,
|
||||
partitionsCompleted INT DEFAULT 0,
|
||||
partitionsFailed INT DEFAULT 0,
|
||||
lastUpdatedAt BIGINT NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (jobId, serverId)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_server_stats_job_id ON search_index_server_stats(jobId);
|
||||
|
||||
-- Create Learning Resource Entity Table
|
||||
CREATE TABLE IF NOT EXISTS learning_resource_entity (
|
||||
id character varying(36) GENERATED ALWAYS AS ((json ->> 'id'::text)) STORED NOT NULL,
|
||||
name character varying(3072) GENERATED ALWAYS AS ((json ->> 'fullyQualifiedName'::text)) STORED,
|
||||
fqnhash character varying(256) NOT NULL,
|
||||
json jsonb NOT NULL,
|
||||
updatedat bigint GENERATED ALWAYS AS (((json ->> 'updatedAt'::text))::bigint) STORED NOT NULL,
|
||||
updatedby character varying(256) GENERATED ALWAYS AS ((json ->> 'updatedBy'::text)) STORED NOT NULL,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnhash)
|
||||
);
|
||||
|
||||
-- DELETE old workflow instances where status is null
|
||||
DELETE FROM workflow_instance_time_series
|
||||
WHERE json->>'status' IS NULL;
|
||||
|
||||
-- DELETE old workflow instance states where status is null
|
||||
DELETE FROM workflow_instance_state_time_series
|
||||
WHERE json->>'status' IS NULL;
|
||||
|
||||
-- Widen entityLink generated column from VARCHAR(255) to TEXT
|
||||
-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
|
||||
ALTER TABLE workflow_instance_time_series ALTER COLUMN entityLink TYPE TEXT;
|
||||
|
||||
-- Add process and vector stage columns to search_index_server_stats table
|
||||
-- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
|
||||
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS processSuccess BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS processFailed BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS vectorSuccess BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS vectorFailed BIGINT DEFAULT 0;
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS vectorWarnings BIGINT DEFAULT 0;
|
||||
|
||||
-- Add entityType column to support per-entity stats tracking
|
||||
-- Stats are now tracked per (jobId, serverId, entityType) instead of (jobId, serverId)
|
||||
ALTER TABLE search_index_server_stats ADD COLUMN IF NOT EXISTS entityType VARCHAR(128) NOT NULL DEFAULT 'unknown';
|
||||
|
||||
-- Drop old unique constraint and index, then create new one with entityType
|
||||
ALTER TABLE search_index_server_stats DROP CONSTRAINT IF EXISTS search_index_server_stats_jobid_serverid_key;
|
||||
DROP INDEX IF EXISTS idx_search_index_server_stats_job_server;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_search_index_server_stats_job_server_entity
|
||||
ON search_index_server_stats (jobId, serverId, entityType);
|
||||
|
||||
-- Remove deprecated columns (entityBuildFailures is redundant - failures are tracked as processFailed)
|
||||
-- sinkTotal and sinkWarnings are not needed
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN IF EXISTS entityBuildFailures;
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN IF EXISTS sinkTotal;
|
||||
ALTER TABLE search_index_server_stats DROP COLUMN IF EXISTS sinkWarnings;
|
||||
|
||||
-- AI Application and LLM Model entities for AI/LLM governance
|
||||
-- Version 1.12.0
|
||||
|
||||
-- Create ai_application_entity table
|
||||
CREATE TABLE IF NOT EXISTS ai_application_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ai_application_name_index ON ai_application_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS ai_application_deleted_index ON ai_application_entity(deleted);
|
||||
|
||||
COMMENT ON TABLE ai_application_entity IS 'AI Application entities';
|
||||
|
||||
-- Create llm_model_entity table
|
||||
CREATE TABLE IF NOT EXISTS llm_model_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS llm_model_name_index ON llm_model_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS llm_model_deleted_index ON llm_model_entity(deleted);
|
||||
|
||||
COMMENT ON TABLE llm_model_entity IS 'LLM Model entities';
|
||||
|
||||
-- Create prompt_template_entity table
|
||||
CREATE TABLE IF NOT EXISTS prompt_template_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS prompt_template_name_index ON prompt_template_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS prompt_template_deleted_index ON prompt_template_entity(deleted);
|
||||
|
||||
COMMENT ON TABLE prompt_template_entity IS 'Prompt Template entities';
|
||||
|
||||
-- Create agent_execution_entity table
|
||||
CREATE TABLE IF NOT EXISTS agent_execution_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
|
||||
agentId VARCHAR(36) GENERATED ALWAYS AS (json->>'agentId') STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
timestamp BIGINT GENERATED ALWAYS AS ((json->>'timestamp')::bigint) STORED NOT NULL,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS agent_execution_agent_index ON agent_execution_entity(agentId);
|
||||
CREATE INDEX IF NOT EXISTS agent_execution_timestamp_index ON agent_execution_entity(timestamp);
|
||||
|
||||
COMMENT ON TABLE agent_execution_entity IS 'AI Agent Execution logs';
|
||||
|
||||
-- Create ai_governance_policy_entity table
|
||||
CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
|
||||
fqnHash VARCHAR(768) NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (fqnHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ai_governance_policy_name_index ON ai_governance_policy_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS ai_governance_policy_deleted_index ON ai_governance_policy_entity(deleted);
|
||||
|
||||
COMMENT ON TABLE ai_governance_policy_entity IS 'AI Governance Policy entities';
|
||||
|
||||
-- Create llm_service_entity table
|
||||
CREATE TABLE IF NOT EXISTS llm_service_entity (
|
||||
id VARCHAR(36) GENERATED ALWAYS AS ((json ->> 'id'::text)) STORED NOT NULL,
|
||||
name VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'name'::text)) STORED NOT NULL,
|
||||
serviceType VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'serviceType'::text)) STORED NOT NULL,
|
||||
json JSONB NOT NULL,
|
||||
updatedAt BIGINT GENERATED ALWAYS AS (((json ->> 'updatedAt'::text))::bigint) STORED NOT NULL,
|
||||
updatedBy VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'updatedBy'::text)) STORED NOT NULL,
|
||||
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
|
||||
deleted BOOLEAN GENERATED ALWAYS AS (((json ->> 'deleted'::text))::boolean) STORED,
|
||||
nameHash VARCHAR(256) NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE (nameHash)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS llm_service_name_index ON llm_service_entity(name);
|
||||
CREATE INDEX IF NOT EXISTS llm_service_type_index ON llm_service_entity(serviceType);
|
||||
CREATE INDEX IF NOT EXISTS llm_service_deleted_index ON llm_service_entity(deleted);
|
||||
|
||||
COMMENT ON TABLE llm_service_entity IS 'LLM Service entities';
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Add search_text column for full-text search on audit log events.
|
||||
-- Populated at write time with searchable content extracted from the change event
|
||||
-- (user name, entity FQN, entity type, service name, field change names and values).
|
||||
-- This avoids scanning the event_json LONGTEXT column at query time.
|
||||
ALTER TABLE audit_log_event ADD COLUMN search_text TEXT DEFAULT NULL;
|
||||
|
||||
-- MySQL built-in FULLTEXT index on InnoDB — no extensions required.
|
||||
-- Supports MATCH() AGAINST() queries in NATURAL LANGUAGE mode.
|
||||
CREATE FULLTEXT INDEX idx_audit_log_search_text ON audit_log_event (search_text);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Add search_text column for full-text search on audit log events.
|
||||
-- Populated at write time with searchable content extracted from the change event
|
||||
-- (user name, entity FQN, entity type, service name, field change names and values).
|
||||
-- This avoids scanning the event_json TEXT column at query time.
|
||||
ALTER TABLE audit_log_event ADD COLUMN IF NOT EXISTS search_text TEXT DEFAULT NULL;
|
||||
|
||||
-- PostgreSQL built-in GIN index on tsvector expression — no extensions required.
|
||||
-- Supports to_tsvector() @@ plainto_tsquery() queries for full-text search.
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_search_text ON audit_log_event USING GIN (to_tsvector('english', coalesce(search_text, '')));
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Placeholder for 1.12.10 MySQL schema changes
|
||||
-- No DDL changes required; this version only carries a data migration
|
||||
-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings).
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Placeholder for 1.12.10 Postgres schema changes
|
||||
-- No DDL changes required; this version only carries a data migration
|
||||
-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings).
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Widen tag_usage.tagFQN (was VARCHAR(256)) toward the fullyQualifiedEntityName contract (max 3072).
|
||||
-- Entity names allow up to 256 chars and nested glossary terms concatenate, so a tag FQN routinely
|
||||
-- exceeds 256 -- which made applyTag inserts and glossaryTerm moveAsync renames fail with
|
||||
-- "value too long". MySQL widens the base column in place (the tagfqn_lower STORED generated column
|
||||
-- and the tagFQN(255)-prefix indexes are unaffected). 512 keeps the full-tagFQN
|
||||
-- idx_tag_usage_target_source within InnoDB's 3072-byte key limit (768 + 1 + 512*4 = 2817 bytes).
|
||||
ALTER TABLE tag_usage MODIFY tagFQN VARCHAR(512) NOT NULL;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Widen tag_usage.tagFQN (was VARCHAR(256)) toward the fullyQualifiedEntityName contract (max 3072).
|
||||
-- Entity names allow up to 256 chars and nested glossary terms concatenate, so a tag FQN routinely
|
||||
-- exceeds 256 -- which made applyTag inserts and glossaryTerm moveAsync renames fail with
|
||||
-- "value too long for type character varying(256)". tagFQN feeds the tagfqn_lower generated column,
|
||||
-- which Postgres will not let us re-type in place, so drop it (cascading its two text_pattern_ops
|
||||
-- indexes), widen tagFQN, then recreate the column and those indexes unchanged. 512 keeps the column
|
||||
-- within MySQL InnoDB's 3072-byte key limit for the full-tagFQN idx_tag_usage_target_source index.
|
||||
ALTER TABLE tag_usage DROP COLUMN IF EXISTS tagfqn_lower;
|
||||
ALTER TABLE tag_usage ALTER COLUMN tagFQN TYPE VARCHAR(512);
|
||||
ALTER TABLE tag_usage ADD COLUMN tagfqn_lower text GENERATED ALWAYS AS (lower(tagFQN)) STORED;
|
||||
ALTER TABLE tag_usage ALTER COLUMN tagfqn_lower SET STATISTICS 500;
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_usage_tagfqn_lower_pattern
|
||||
ON tag_usage (tagfqn_lower text_pattern_ops);
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_usage_tagfqn_prefix_covering
|
||||
ON tag_usage (source, tagfqn_lower text_pattern_ops)
|
||||
INCLUDE (targetFQNHash, labelType, state);
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Keyset pagination indexes for time-series tables used during search index reindexing
|
||||
CREATE INDEX idx_entity_extension_ts_keyset ON entity_extension_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX idx_report_data_ts_keyset ON report_data_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX idx_data_quality_data_ts_keyset ON data_quality_data_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX idx_test_case_resolution_status_ts_keyset ON test_case_resolution_status_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX idx_query_cost_ts_keyset ON query_cost_time_series(timestamp, entityFQNHash);
|
||||
|
||||
-- Index for listLastTestCaseResultsForTestSuite / listLastTestCaseResult performance (MySQL).
|
||||
-- Supports "latest row per entityFQNHash" in data_quality_data_time_series.
|
||||
-- Use with FORCE INDEX (idx_entity_timestamp_desc) in TestCaseResultTimeSeriesDAO for MySQL.
|
||||
ALTER TABLE data_quality_data_time_series ADD KEY idx_entity_timestamp_desc (entityFQNHash, timestamp DESC);
|
||||
-- Add entityStatus generated column to glossary_term_entity table for efficient filtering
|
||||
-- This supports the entityStatus filtering in the search API endpoint
|
||||
ALTER TABLE glossary_term_entity
|
||||
ADD COLUMN entityStatus VARCHAR(32)
|
||||
GENERATED ALWAYS AS (json_unquote(json_extract(json, '$.entityStatus')))
|
||||
STORED;
|
||||
|
||||
-- Add index for efficient entityStatus filtering
|
||||
CREATE INDEX idx_glossary_term_entity_status ON glossary_term_entity (entityStatus);
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Keyset pagination indexes for time-series tables used during search index reindexing
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_extension_ts_keyset
|
||||
ON entity_extension_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_report_data_ts_keyset
|
||||
ON report_data_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_quality_data_ts_keyset
|
||||
ON data_quality_data_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_test_case_resolution_status_ts_keyset
|
||||
ON test_case_resolution_status_time_series(timestamp, entityFQNHash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_query_cost_ts_keyset
|
||||
ON query_cost_time_series(timestamp, entityFQNHash);
|
||||
|
||||
-- Index for listLastTestCaseResultsForTestSuite / listLastTestCaseResult performance.
|
||||
-- Supports "latest row per entityFQNHash" in data_quality_data_time_series.
|
||||
CREATE INDEX IF NOT EXISTS idx_entity_timestamp_desc
|
||||
ON data_quality_data_time_series(entityFQNHash, timestamp DESC);
|
||||
-- Add entityStatus generated column to glossary_term_entity table for efficient filtering
|
||||
-- This supports the entityStatus filtering in the search API endpoint
|
||||
ALTER TABLE glossary_term_entity
|
||||
ADD COLUMN IF NOT EXISTS entityStatus VARCHAR(32)
|
||||
GENERATED ALWAYS AS (json ->> 'entityStatus')
|
||||
STORED;
|
||||
|
||||
-- Add index for efficient entityStatus filtering
|
||||
CREATE INDEX IF NOT EXISTS idx_glossary_term_entity_status ON glossary_term_entity (entityStatus);
|
||||
@@ -0,0 +1,4 @@
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = JSON_REMOVE(json, '$.sourceConfig.config.threadCount')
|
||||
WHERE pipelineType = 'profiler'
|
||||
AND JSON_EXTRACT(json, '$.sourceConfig.config.threadCount') = 5;
|
||||
@@ -0,0 +1,109 @@
|
||||
-- MCP OAuth 2.0 tables
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_clients (
|
||||
id CHAR(36) PRIMARY KEY DEFAULT (UUID()),
|
||||
client_id VARCHAR(255) UNIQUE NOT NULL,
|
||||
client_secret_encrypted TEXT,
|
||||
client_name VARCHAR(255),
|
||||
redirect_uris JSON NOT NULL DEFAULT ('[]'),
|
||||
grant_types JSON NOT NULL DEFAULT ('["authorization_code", "refresh_token"]'),
|
||||
token_endpoint_auth_method VARCHAR(50) NOT NULL DEFAULT 'client_secret_post',
|
||||
scopes JSON NOT NULL DEFAULT ('["read", "write"]'),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_clients_client_id_check CHECK (CHAR_LENGTH(client_id) > 0),
|
||||
INDEX idx_oauth_clients_client_id (client_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_authorization_codes (
|
||||
id CHAR(36) PRIMARY KEY DEFAULT (UUID()),
|
||||
code VARCHAR(512) UNIQUE NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
code_challenge VARCHAR(255),
|
||||
code_challenge_method VARCHAR(10),
|
||||
redirect_uri TEXT NOT NULL,
|
||||
scopes JSON NOT NULL DEFAULT ('[]'),
|
||||
expires_at BIGINT NOT NULL,
|
||||
used BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_authorization_codes_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_authorization_codes_expires_check CHECK (expires_at > 0),
|
||||
CONSTRAINT oauth_authorization_codes_challenge_method_check CHECK (
|
||||
code_challenge_method IS NULL OR code_challenge_method IN ('S256')
|
||||
),
|
||||
INDEX idx_oauth_authz_codes_code (code),
|
||||
INDEX idx_oauth_authz_codes_client_id (client_id),
|
||||
INDEX idx_oauth_authz_codes_expires_at (expires_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_access_tokens (
|
||||
id CHAR(36) PRIMARY KEY DEFAULT (UUID()),
|
||||
token_hash VARCHAR(255) UNIQUE NOT NULL,
|
||||
access_token_encrypted TEXT NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
scopes JSON NOT NULL DEFAULT ('[]'),
|
||||
expires_at BIGINT NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_access_tokens_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_access_tokens_expires_check CHECK (expires_at > 0),
|
||||
INDEX idx_oauth_access_tokens_hash (token_hash),
|
||||
INDEX idx_oauth_access_tokens_client_id (client_id),
|
||||
INDEX idx_oauth_access_tokens_expires_at (expires_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_refresh_tokens (
|
||||
id CHAR(36) PRIMARY KEY DEFAULT (UUID()),
|
||||
token_hash VARCHAR(255) UNIQUE NOT NULL,
|
||||
refresh_token_encrypted TEXT NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
scopes JSON NOT NULL DEFAULT ('[]'),
|
||||
expires_at BIGINT NOT NULL,
|
||||
revoked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_refresh_tokens_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_refresh_tokens_expires_check CHECK (expires_at > 0),
|
||||
INDEX idx_oauth_refresh_tokens_hash (token_hash),
|
||||
INDEX idx_oauth_refresh_tokens_client_id (client_id),
|
||||
INDEX idx_oauth_refresh_tokens_revoked (revoked),
|
||||
INDEX idx_oauth_refresh_tokens_expires_at (expires_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mcp_pending_auth_requests (
|
||||
id CHAR(36) PRIMARY KEY DEFAULT (UUID()),
|
||||
auth_request_id VARCHAR(64) UNIQUE NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
code_challenge VARCHAR(255) NOT NULL,
|
||||
code_challenge_method VARCHAR(10) NOT NULL DEFAULT 'S256',
|
||||
redirect_uri TEXT NOT NULL,
|
||||
mcp_state VARCHAR(255),
|
||||
scopes JSON,
|
||||
pac4j_state VARCHAR(64),
|
||||
pac4j_nonce VARCHAR(255),
|
||||
pac4j_code_verifier VARCHAR(255),
|
||||
expires_at BIGINT NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX idx_mcp_pending_auth_request_id (auth_request_id),
|
||||
INDEX idx_mcp_pending_auth_expires (expires_at),
|
||||
INDEX idx_mcp_pending_auth_pac4j_state (pac4j_state)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
-- Incremental Search Retry Queue
|
||||
-- Stores failed live-indexing operations for async background catch-up.
|
||||
-- Keep this table intentionally minimal: entityId, entityFqn, failureReason, status.
|
||||
CREATE TABLE IF NOT EXISTS search_index_retry_queue (
|
||||
entityId VARCHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
|
||||
entityFqn VARCHAR(1024) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
|
||||
failureReason LONGTEXT,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
entityType VARCHAR(256) NOT NULL DEFAULT '',
|
||||
retryCount INT NOT NULL DEFAULT 0,
|
||||
claimedAt TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (entityId, entityFqn),
|
||||
INDEX idx_search_index_retry_queue_status (status),
|
||||
INDEX idx_search_index_retry_queue_claimed (claimedAt)
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
UPDATE ingestion_pipeline_entity
|
||||
SET json = (json::jsonb #- '{sourceConfig,config,threadCount}')::json
|
||||
WHERE pipelineType = 'profiler'
|
||||
AND json->'sourceConfig'->'config'->>'threadCount' IS NOT NULL
|
||||
AND (json->'sourceConfig'->'config'->>'threadCount')::numeric = 5;
|
||||
@@ -0,0 +1,118 @@
|
||||
-- MCP OAuth 2.0 tables
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_clients (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
client_id VARCHAR(255) UNIQUE NOT NULL,
|
||||
client_secret_encrypted TEXT,
|
||||
client_name VARCHAR(255),
|
||||
redirect_uris JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
grant_types JSONB NOT NULL DEFAULT '["authorization_code", "refresh_token"]'::jsonb,
|
||||
token_endpoint_auth_method VARCHAR(50) NOT NULL DEFAULT 'client_secret_post',
|
||||
scopes JSONB NOT NULL DEFAULT '["read", "write"]'::jsonb,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_clients_client_id_check CHECK (char_length(client_id) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_clients_client_id ON oauth_clients(client_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_authorization_codes (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
code VARCHAR(512) UNIQUE NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
code_challenge VARCHAR(255),
|
||||
code_challenge_method VARCHAR(10),
|
||||
redirect_uri TEXT NOT NULL,
|
||||
scopes JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
expires_at BIGINT NOT NULL,
|
||||
used BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_authorization_codes_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_authorization_codes_expires_check CHECK (expires_at > 0),
|
||||
CONSTRAINT oauth_authorization_codes_challenge_method_check CHECK (
|
||||
code_challenge_method IS NULL OR code_challenge_method IN ('S256')
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_authz_codes_code ON oauth_authorization_codes(code);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_authz_codes_client_id ON oauth_authorization_codes(client_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_authz_codes_expires_at ON oauth_authorization_codes(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_access_tokens (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
token_hash VARCHAR(255) UNIQUE NOT NULL,
|
||||
access_token_encrypted TEXT NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
scopes JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
expires_at BIGINT NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_access_tokens_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_access_tokens_expires_check CHECK (expires_at > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_access_tokens_hash ON oauth_access_tokens(token_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_access_tokens_client_id ON oauth_access_tokens(client_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_access_tokens_expires_at ON oauth_access_tokens(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_refresh_tokens (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
token_hash VARCHAR(255) UNIQUE NOT NULL,
|
||||
refresh_token_encrypted TEXT NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255) NOT NULL,
|
||||
scopes JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
expires_at BIGINT NOT NULL,
|
||||
revoked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT oauth_refresh_tokens_fk_client
|
||||
FOREIGN KEY (client_id) REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
|
||||
CONSTRAINT oauth_refresh_tokens_expires_check CHECK (expires_at > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_refresh_tokens_hash ON oauth_refresh_tokens(token_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_refresh_tokens_client_id ON oauth_refresh_tokens(client_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_refresh_tokens_revoked ON oauth_refresh_tokens(revoked);
|
||||
CREATE INDEX IF NOT EXISTS idx_oauth_refresh_tokens_expires_at ON oauth_refresh_tokens(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mcp_pending_auth_requests (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
auth_request_id VARCHAR(64) UNIQUE NOT NULL,
|
||||
client_id VARCHAR(255) NOT NULL,
|
||||
code_challenge VARCHAR(255) NOT NULL,
|
||||
code_challenge_method VARCHAR(10) NOT NULL DEFAULT 'S256',
|
||||
redirect_uri TEXT NOT NULL,
|
||||
mcp_state VARCHAR(255),
|
||||
scopes JSONB,
|
||||
pac4j_state VARCHAR(64),
|
||||
pac4j_nonce VARCHAR(255),
|
||||
pac4j_code_verifier VARCHAR(255),
|
||||
expires_at BIGINT NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mcp_pending_auth_request_id ON mcp_pending_auth_requests(auth_request_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_mcp_pending_auth_expires ON mcp_pending_auth_requests(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_mcp_pending_auth_pac4j_state ON mcp_pending_auth_requests(pac4j_state);
|
||||
|
||||
-- Incremental Search Retry Queue
|
||||
-- Stores failed live-indexing operations for async background catch-up.
|
||||
-- Keep this table intentionally minimal: entityId, entityFqn, failureReason, status.
|
||||
CREATE TABLE IF NOT EXISTS search_index_retry_queue (
|
||||
entityId VARCHAR(36) NOT NULL DEFAULT '',
|
||||
entityFqn VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
failureReason TEXT,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
entityType VARCHAR(256) NOT NULL DEFAULT '',
|
||||
retryCount INT NOT NULL DEFAULT 0,
|
||||
claimedAt TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (entityId, entityFqn)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_retry_queue_status
|
||||
ON search_index_retry_queue(status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_index_retry_queue_claimed
|
||||
ON search_index_retry_queue(claimedAt);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Delete SearchIndexingApplication from entity_relationship (via installed_apps)
|
||||
DELETE er FROM entity_relationship er JOIN installed_apps ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
|
||||
|
||||
-- Delete SearchIndexingApplication from entity_relationship (via apps_marketplace)
|
||||
DELETE er FROM entity_relationship er JOIN apps_marketplace ia ON er.fromId = ia.id OR er.toId = ia.id WHERE ia.name = 'SearchIndexingApplication';
|
||||
|
||||
-- Delete SearchIndexingApplication from installed_apps
|
||||
DELETE FROM installed_apps WHERE name = 'SearchIndexingApplication';
|
||||
|
||||
-- Delete SearchIndexingApplication from apps_marketplace
|
||||
DELETE FROM apps_marketplace WHERE name = 'SearchIndexingApplication';
|
||||
|
||||
-- Delete SearchIndexingApplication past runs
|
||||
DELETE FROM apps_extension_time_series WHERE appname = 'SearchIndexingApplication';
|
||||
@@ -0,0 +1 @@
|
||||
-- Placeholder for 1.12.5 MySQL schema changes
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user