wehub-resource-sync
bf2343b7e4
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
389 lines
14 KiB
JSON
389 lines
14 KiB
JSON
{
|
|
"pipelines": [{
|
|
"name": "presto_etl",
|
|
"displayName": "Presto ETL",
|
|
"description": "Presto ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=presto_etl",
|
|
"scheduleInterval": "* * * * *",
|
|
"tasks": [
|
|
{
|
|
"name": "presto_task",
|
|
"displayName": "Presto Task",
|
|
"description": "Airflow operator to perform ETL on presto tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "dim_address_etl",
|
|
"displayName": "dim_address etl",
|
|
"description": "dim_address ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_address_etl",
|
|
"scheduleInterval": "5 * * * *",
|
|
"tasks": [{
|
|
"name": "dim_address_task",
|
|
"displayName": "dim_address Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_address table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_address_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "dim_user_etl",
|
|
"displayName": "dim_user etl",
|
|
"description": "dim_user ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_user_etl",
|
|
"tasks": [{
|
|
"name": "dim_user_task",
|
|
"displayName": "dim_user Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_user table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_user_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "dim_location_etl",
|
|
"displayName": "dim_location etl",
|
|
"description": "diim_location ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_address_etl",
|
|
"tasks": [{
|
|
"name": "dim_location_task",
|
|
"displayName": "dim_location Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_location table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_location_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "dim_product_etl",
|
|
"displayName": "dim_product etl",
|
|
"description": "diim_product ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_address_etl",
|
|
"tasks": [{
|
|
"name": "dim_product_task",
|
|
"displayName": "dim_product Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_product table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_product_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "trino_etl",
|
|
"displayName": "Trino ETL",
|
|
"description": "Trino ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=trino_etl",
|
|
"scheduleInterval": "@once",
|
|
"tasks": [{
|
|
"name": "trino_task",
|
|
"displayName": "Trino Task",
|
|
"description": "Airflow operator to perform ETL on trino tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "TrinoOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "hive_etl",
|
|
"displayName": "Hive ETL",
|
|
"description": "Hive ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=hive_etl",
|
|
"tasks": [{
|
|
"name": "hive_create_table",
|
|
"displayName": "Hive Create Table",
|
|
"description": "Hive Create Table Task",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=hive_create_table",
|
|
"downstreamTasks": ["assert_table_exits"],
|
|
"taskType": "HiveOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "snowflake_etl",
|
|
"displayName": "Snowflake ETL",
|
|
"description": "Snowflake ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=snowflake_etl",
|
|
"tasks": [{
|
|
"name": "snowflake_task",
|
|
"displayName": "Snowflake Task",
|
|
"description": "Airflow operator to perform ETL on snowflake tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "SnowflakeOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}]
|
|
},
|
|
{
|
|
"name": "real_time_metrics",
|
|
"displayName": "Real-time Metrics Aggregation",
|
|
"description": "Real-time metrics aggregation pipeline running every 15 minutes",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=real_time_metrics",
|
|
"scheduleInterval": "*/15 * * * *",
|
|
"tasks": [{
|
|
"name": "metrics_aggregation_task",
|
|
"displayName": "Metrics Aggregation Task",
|
|
"description": "Airflow operator to aggregate streaming metrics",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=metrics_aggregation_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}]
|
|
},
|
|
{
|
|
"name": "ml_feature_pipeline",
|
|
"displayName": "ML Feature Engineering Pipeline",
|
|
"description": "Machine learning feature engineering pipeline running twice daily",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=ml_feature_pipeline",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"tasks": [{
|
|
"name": "feature_engineering_task",
|
|
"displayName": "Feature Engineering Task",
|
|
"description": "Airflow operator to generate ML features from order data",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=feature_engineering_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_staging_shopify",
|
|
"displayName": "DBT Staging Shopify",
|
|
"description": "DBT pipeline for staging shopify raw data - runs daily",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_staging_shopify",
|
|
"scheduleInterval": "0 1 * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_run_staging",
|
|
"displayName": "DBT Run Staging Models",
|
|
"description": "Execute dbt run for staging models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_staging_shopify",
|
|
"downstreamTasks": ["dbt_test_staging"],
|
|
"taskType": "dbtRunTask"
|
|
},
|
|
{
|
|
"name": "dbt_test_staging",
|
|
"displayName": "DBT Test Staging Models",
|
|
"description": "Execute dbt tests for staging models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_staging_shopify",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtTestTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_transform_orders",
|
|
"displayName": "DBT Transform Orders",
|
|
"description": "DBT pipeline for transforming order data into fact tables - runs daily",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_transform_orders",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_run_orders",
|
|
"displayName": "DBT Run Order Transformations",
|
|
"description": "Execute dbt run for order fact tables",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_transform_orders",
|
|
"downstreamTasks": ["dbt_test_orders"],
|
|
"taskType": "dbtRunTask"
|
|
},
|
|
{
|
|
"name": "dbt_test_orders",
|
|
"displayName": "DBT Test Order Models",
|
|
"description": "Execute dbt tests for order models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_transform_orders",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtTestTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_analytics_customers",
|
|
"displayName": "DBT Customer Analytics",
|
|
"description": "DBT pipeline for customer analytics and aggregations - runs daily",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_analytics_customers",
|
|
"scheduleInterval": "0 4 * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_build_analytics",
|
|
"displayName": "DBT Build Customer Analytics",
|
|
"description": "Execute dbt build for customer analytics models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_analytics_customers",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtBuildTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_snapshot_inventory",
|
|
"displayName": "DBT Snapshot Inventory",
|
|
"description": "DBT snapshot pipeline for tracking inventory changes - runs hourly",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_snapshot_inventory",
|
|
"scheduleInterval": "0 * * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_snapshot_task",
|
|
"displayName": "DBT Snapshot",
|
|
"description": "Execute dbt snapshot for inventory tracking",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_snapshot_inventory",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtSnapshotTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_test_data_quality",
|
|
"displayName": "DBT Data Quality Tests",
|
|
"description": "DBT data quality testing pipeline - runs 4 times daily",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_test_data_quality",
|
|
"scheduleInterval": "0 */6 * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_test_all",
|
|
"displayName": "DBT Test All Models",
|
|
"description": "Execute comprehensive dbt tests across all models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_test_data_quality",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtTestTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "dbt_ml_features",
|
|
"displayName": "DBT ML Feature Generation",
|
|
"description": "DBT pipeline for generating ML features - runs twice daily",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_ml_features",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"service": "sample_dbtcloud",
|
|
"tasks": [{
|
|
"name": "dbt_run_ml_features",
|
|
"displayName": "DBT Run ML Features",
|
|
"description": "Execute dbt run for ML feature generation",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_ml_features",
|
|
"downstreamTasks": ["dbt_test_ml_features"],
|
|
"taskType": "dbtRunTask"
|
|
},
|
|
{
|
|
"name": "dbt_test_ml_features",
|
|
"displayName": "DBT Test ML Features",
|
|
"description": "Execute dbt tests for ML feature models",
|
|
"sourceUrl": "https://cloud.getdbt.com/deploy/123456/projects/654321/runs/dbt_ml_features",
|
|
"downstreamTasks": [],
|
|
"taskType": "dbtTestTask"
|
|
}]
|
|
},
|
|
{
|
|
"name": "provider_address_texas_pipeline",
|
|
"displayName": "Provider Address Texas Pipeline",
|
|
"description": "Search relevancy fixture pipeline for loading provider address Texas reference data.",
|
|
"sourceUrl": "https://airflow.example.com/dags/provider_address_texas_pipeline",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"service": "sample_airflow",
|
|
"tasks": [{
|
|
"name": "load_provider_address_texas",
|
|
"displayName": "Load Provider Address Texas",
|
|
"description": "Load the provider address Texas fixture table.",
|
|
"sourceUrl": "https://airflow.example.com/dags/provider_address_texas_pipeline/tasks/load_provider_address_texas",
|
|
"downstreamTasks": [],
|
|
"taskType": "PythonOperator"
|
|
}]
|
|
},
|
|
{
|
|
"name": "customer_profiles_pipeline",
|
|
"displayName": "Customer Profiles Pipeline",
|
|
"description": "Search relevancy fixture pipeline for loading customer profiles reference data.",
|
|
"sourceUrl": "https://airflow.example.com/dags/customer_profiles_pipeline",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"service": "sample_airflow",
|
|
"tasks": [{
|
|
"name": "load_customer_profiles",
|
|
"displayName": "Load Customer Profiles",
|
|
"description": "Load the customer profiles fixture table.",
|
|
"sourceUrl": "https://airflow.example.com/dags/customer_profiles_pipeline/tasks/load_customer_profiles",
|
|
"downstreamTasks": [],
|
|
"taskType": "PythonOperator"
|
|
}]
|
|
}
|
|
]
|
|
}
|