wehub/open-metadata--openmetadata
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Details
Java Checkstyle / java-checkstyle (push) Has been cancelled
Details
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Details
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:35:45 +08:00
commit bf2343b7e4
16049 changed files with 3531137 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bootstrap/sql/migrations/native/1.1.5/mysql/postDataMigrationSQLScript.sql
+50
View File
@@ -0,0 +1,50 @@
START TRANSACTION;
-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
select ids FROM (
SELECT
(json ->> '$.id') AS ids,
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
FROM (
SELECT
*
FROM report_data_time_series rdts
WHERE json ->> '$.reportDataType' = 'WebAnalyticEntityViewReportData'
) duplicates
ORDER BY `date` DESC, `timestamp` DESC
) as dense_ranked
WHERE denseRank != 1
);
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
select ids FROM (
SELECT
(json ->> '$.id') AS ids,
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
FROM (
SELECT
*
FROM report_data_time_series rdts
WHERE json ->> '$.reportDataType' = 'EntityReportData'
) duplicates
ORDER BY `date` DESC, `timestamp` DESC
) as dense_ranked
WHERE denseRank != 1
);
DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
select ids FROM (
SELECT
(json ->> '$.id') AS ids,
DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
FROM (
SELECT
*
FROM report_data_time_series rdts
WHERE json ->> '$.reportDataType' = 'WebAnalyticUserActivityReportData'
) duplicates
ORDER BY `date` DESC, `timestamp` DESC
) as dense_ranked
WHERE denseRank != 1
);
COMMIT;
bootstrap/sql/migrations/native/1.1.5/mysql/schemaChanges.sql
+120
View File
@@ -0,0 +1,120 @@
-- Update table and column profile timestamps to be in milliseconds
UPDATE entity_extension_time_series
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.timestamp'),
'$.timestamp',
JSON_EXTRACT(json, '$.timestamp') * 1000
)
WHERE
extension in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
;
START TRANSACTION;
-- Create report data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS report_data_time_series (
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
date DATE GENERATED ALWAYS AS (FROM_UNIXTIME((json ->> '$.timestamp') DIV 1000)) NOT NULL,
INDEX report_data_time_series_point_ts (timestamp),
INDEX report_data_time_series_date (date)
);
INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
DELETE FROM entity_extension_time_series
WHERE extension = 'reportData.reportDataResult';
COMMIT;
START TRANSACTION;
-- Create profiler data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS profiler_data_time_series (
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
operation VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.operation') NULL,
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
UNIQUE profiler_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, operation, timestamp),
INDEX profiler_data_time_series_combined_id_ts (extension, timestamp)
);
INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
DELETE FROM entity_extension_time_series
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
COMMIT;
START TRANSACTION;
-- Create data quality data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
UNIQUE data_quality_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, timestamp),
INDEX data_quality_data_time_series_combined_id_ts (extension, timestamp)
);
INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series
WHERE extension = 'testCase.testCaseResult';
DELETE FROM entity_extension_time_series
WHERE extension = 'testCase.testCaseResult';
COMMIT;
ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin,
ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
ALTER TABLE field_relationship MODIFY COLUMN fromFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN toFQNHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE thread_entity MODIFY COLUMN entityLink VARCHAR(3072) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL;
ALTER TABLE event_subscription_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE ingestion_pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE bot_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE user_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE team_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE role_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE policy_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE classification MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE tag MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE tag_usage MODIFY COLUMN tagFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN targetFQNHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE glossary_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE glossary_term_entity MODIFY fqnHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE type_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE web_analytic_event MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE data_insight_chart MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE kpi_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE test_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE test_connection_definition MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE test_suite MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE test_case MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE dbservice_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE messaging_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE metadata_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE pipeline_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE dashboard_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE mlmodel_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE storage_service_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE database_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE database_schema_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE table_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE dashboard_data_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE dashboard_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE chart_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE ml_model_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE metric_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE query_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE report_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE storage_container_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
ALTER TABLE topic_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
bootstrap/sql/migrations/native/1.1.5/postgres/postDataMigrationSQLScript.sql
+53
View File
@@ -0,0 +1,53 @@
BEGIN;
-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
select ids FROM (
SELECT
(json ->> 'id') AS ids,
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
FROM (
SELECT
*,
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
FROM report_data_time_series rdts
WHERE json ->> 'reportDataType' = 'WebAnalyticEntityViewReportData'
) duplicates
ORDER BY date DESC, timestamp DESC
) as dense_ranked
WHERE denseRank != 1
);
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
select ids FROM (
SELECT
(json ->> 'id') AS ids,
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
FROM (
SELECT
*,
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
FROM report_data_time_series rdts
WHERE json ->> 'reportDataType' = 'EntityReportData'
) duplicates
ORDER BY date DESC, timestamp DESC
) as dense_ranked
WHERE denseRank != 1
);
DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
select ids FROM (
SELECT
(json ->> 'id') AS ids,
DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
FROM (
SELECT
*,
DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
FROM report_data_time_series rdts
WHERE json ->> 'reportDataType' = 'WebAnalyticUserActivityReportData'
) duplicates
ORDER BY date DESC, timestamp DESC
) as dense_ranked
WHERE denseRank != 1
);
COMMIT;
bootstrap/sql/migrations/native/1.1.5/postgres/schemaChanges.sql
+83
View File
@@ -0,0 +1,83 @@
-- Update table and column profile timestamps to be in milliseconds
UPDATE entity_extension_time_series
SET json = jsonb_set(
json,
'{timestamp}',
to_jsonb(cast(json#>'{timestamp}' as int8) *1000)
)
WHERE
extension in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
;
BEGIN;
-- Run the following SQL to update the schema in a transaction
-- Create report data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS report_data_time_series (
entityFQNHash VARCHAR(768),
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL
);
CREATE INDEX IF NOT EXISTS report_data_time_series_point_ts ON report_data_time_series (timestamp);
INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
DELETE FROM entity_extension_time_series
WHERE extension = 'reportData.reportDataResult';
COMMIT;
BEGIN;
-- Create profiler data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS profiler_data_time_series (
entityFQNHash VARCHAR(768),
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
operation VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'operation')::text) STORED NULL,
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
CONSTRAINT profiler_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, operation, timestamp)
);
CREATE INDEX IF NOT EXISTS profiler_data_time_series_combined_id_ts ON profiler_data_time_series (extension, timestamp);
INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
DELETE FROM entity_extension_time_series
WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
COMMIT;
BEGIN;
-- Create profiler data time series table and move data from entity_extension_time_series
CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
entityFQNHash VARCHAR(768),
extension VARCHAR(256) NOT NULL,
jsonSchema VARCHAR(256) NOT NULL,
json JSON NOT NULL,
timestamp BIGINT CHECK (timestamp > 0) GENERATED ALWAYS AS ((json ->> 'timestamp')::bigint) STORED NOT NULL,
CONSTRAINT data_quality_data_time_series_unique_hash_extension_ts UNIQUE(entityFQNHash, extension, timestamp)
);
CREATE INDEX IF NOT EXISTS data_quality_data_time_series_combined_id_ts ON data_quality_data_time_series (extension, timestamp);
INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
SELECT entityFQNHash, extension, jsonSchema, json
FROM entity_extension_time_series
WHERE extension = 'testCase.testCaseResult';
DELETE FROM entity_extension_time_series
WHERE extension = 'testCase.testCaseResult';
COMMIT;
ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQNHash TYPE VARCHAR(768), ALTER COLUMN jsonSchema TYPE VARCHAR(50) , ALTER COLUMN extension TYPE VARCHAR(100) ,
ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
ALTER TABLE field_relationship ALTER COLUMN fromFQNHash TYPE VARCHAR(768), ALTER COLUMN toFQNHash TYPE VARCHAR(768);
ALTER TABLE thread_entity ALTER COLUMN entityLink TYPE VARCHAR(3072);
ALTER TABLE tag_usage ALTER COLUMN tagFQNHash TYPE VARCHAR(768), ALTER COLUMN targetFQNHash TYPE VARCHAR(768);
ALTER TABLE test_suite ALTER COLUMN fqnHash TYPE VARCHAR(768);