marquez
marquez copied to clipboard
Published
20 hours ago •
MarquezProject
MarquezProject
Add column level lineage endpoint
OpenLineage has a column-level lineage facet. Marquez could leverage this to implement a column-level lineage endpoint. Existing:
GET /lineage?nodeId=dataset:food_delivery:public.delivery_7_days
returns all the reachable nodes up to a given number of hops in the graph:
{
"graph": [{
"id": "dataset:food_delivery:public.categories",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.categories"
},
"type": "DB_TABLE",
"name": "public.categories",
"physicalName": "public.categories",
"createdAt": "2021-03-09T02:33:18.468719Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the category."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the category."
}, {
"name": "menu_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the menu related to the category."
}, {
"name": "description",
"type": "TEXT",
"tags": [],
"description": "The description of the category."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:03:09.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:etl_orders.etl_categories",
"destination": "dataset:food_delivery:public.categories"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.categories",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}]
}, {
"id": "dataset:food_delivery:public.customers",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.customers"
},
"type": "DB_TABLE",
"name": "public.customers",
"physicalName": "public.customers",
"createdAt": "2021-03-09T02:33:20.092476Z",
"updatedAt": "2022-08-04T05:18:52.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the customer."
}, {
"name": "created_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the customer was created."
}, {
"name": "updated_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the customer was updated."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the customer."
}, {
"name": "email",
"type": "VARCHAR",
"tags": [],
"description": "The email address of the customer."
}, {
"name": "address",
"type": "VARCHAR",
"tags": [],
"description": "The address of the customer."
}, {
"name": "phone",
"type": "VARCHAR",
"tags": [],
"description": "The phone number of the customer."
}, {
"name": "city_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the city related to the customer."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:07:21.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl.etl_customers",
"destination": "dataset:food_delivery:public.customers"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:analytics.orders_popular_day_of_week"
}, {
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:emails.email_discounts"
}, {
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}]
}, {
"id": "dataset:food_delivery:public.delivery_7_days",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.delivery_7_days"
},
"type": "DB_TABLE",
"name": "public.delivery_7_days",
"physicalName": "public.delivery_7_days",
"createdAt": "2021-03-09T02:33:21.693799Z",
"updatedAt": "2022-08-04T05:15:38.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "order_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the order."
}, {
"name": "order_placed_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was placed."
}, {
"name": "order_dispatched_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was dispatched."
}, {
"name": "order_delivered_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was delivered."
}, {
"name": "customer_email",
"type": "VARCHAR",
"tags": [],
"description": "The email of the customer."
}, {
"name": "menu_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the menu related to the order."
}, {
"name": "menu_item_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the menu item related to the order."
}, {
"name": "category_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of category related to the order."
}, {
"name": "discount_id",
"type": "VARCHAR",
"tags": [],
"description": "The ID of the discount applied to the order."
}, {
"name": "city_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the city related to the order."
}, {
"name": "driver_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the driver related to the order."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:10:50.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl.etl_delivery_7_days",
"destination": "dataset:food_delivery:public.delivery_7_days"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.delivery_7_days",
"destination": "job:food_delivery:analytics.delivery_times_7_days"
}]
}, {
"id": "dataset:food_delivery:public.discounts",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.discounts"
},
"type": "DB_TABLE",
"name": "public.discounts",
"physicalName": "public.discounts",
"createdAt": "2021-03-09T02:33:22.070729Z",
"updatedAt": "2022-08-04T05:18:37.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the discount."
}, {
"name": "amount_off",
"type": "INTEGER",
"tags": [],
"description": "The amount of the discount."
}, {
"name": "customer_email",
"type": "VARCHAR",
"tags": [],
"description": "The email of the customer."
}, {
"name": "starts_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the discount starts."
}, {
"name": "ends_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the discount ends."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:15:38.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:analytics.delivery_times_7_days",
"destination": "dataset:food_delivery:public.discounts"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.discounts",
"destination": "job:food_delivery:emails.email_discounts"
}]
}, {
"id": "dataset:food_delivery:public.drivers",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.drivers"
},
"type": "DB_TABLE",
"name": "public.drivers",
"physicalName": "public.drivers",
"createdAt": "2021-03-09T02:33:21.253785Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the driver."
}, {
"name": "created_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the driver was created."
}, {
"name": "updated_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the driver was updated."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the driver."
}, {
"name": "email",
"type": "VARCHAR",
"tags": [],
"description": "The email of the driver."
}, {
"name": "phone",
"type": "VARCHAR",
"tags": [],
"description": "The phone number of the driver."
}, {
"name": "car_make",
"type": "VARCHAR",
"tags": [],
"description": "The make of the car."
}, {
"name": "car_model",
"type": "VARCHAR",
"tags": [],
"description": "The model of the car."
}, {
"name": "car_year",
"type": "VARCHAR",
"tags": [],
"description": "The year of the car."
}, {
"name": "car_color",
"type": "VARCHAR",
"tags": [],
"description": "The color of the car."
}, {
"name": "car_license_plate",
"type": "VARCHAR",
"tags": [],
"description": "The license plate number of the car."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:06:40.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl.etl_drivers",
"destination": "dataset:food_delivery:public.drivers"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.drivers",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}]
}, {
"id": "dataset:food_delivery:public.menu_items",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.menu_items"
},
"type": "DB_TABLE",
"name": "public.menu_items",
"physicalName": "public.menu_items",
"createdAt": "2021-03-09T02:33:18.959288Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the menu item."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the menu item."
}, {
"name": "price",
"type": "INTEGER",
"tags": [],
"description": "The price of the menu item."
}, {
"name": "category_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the category related to the item."
}, {
"name": "description",
"type": "TEXT",
"tags": [],
"description": "The description of the menu item."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:03:24.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:etl_orders.etl_menu_items",
"destination": "dataset:food_delivery:public.menu_items"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.menu_items",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}]
}, {
"id": "dataset:food_delivery:public.menus",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.menus"
},
"type": "DB_TABLE",
"name": "public.menus",
"physicalName": "public.menus",
"createdAt": "2021-03-09T02:33:17.948768Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the menu."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the menu."
}, {
"name": "restaurant_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the restaurant related to the menu."
}, {
"name": "description",
"type": "TEXT",
"tags": [],
"description": "The description of the menu."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:03:06.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl_orders.etl_menus",
"destination": "dataset:food_delivery:public.menus"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.menus",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}]
}, {
"id": "dataset:food_delivery:public.order_status",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.order_status"
},
"type": "DB_TABLE",
"name": "public.order_status",
"physicalName": "public.order_status",
"createdAt": "2021-03-09T02:33:20.488093Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the order status."
}, {
"name": "transitioned_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order status was transitioned."
}, {
"name": "status",
"type": "VARCHAR",
"tags": [],
"description": "The status of the order status."
}, {
"name": "order_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the order related to the order status."
}, {
"name": "customer_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the customer related to the order status."
}, {
"name": "restaurant_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the restaurant related to the order status."
}, {
"name": "driver_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the driver related to the order status."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:07:05.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:etl.etl_order_status",
"destination": "dataset:food_delivery:public.order_status"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.order_status",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}]
}, {
"id": "dataset:food_delivery:public.orders",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.orders"
},
"type": "DB_TABLE",
"name": "public.orders",
"physicalName": "public.orders",
"createdAt": "2021-03-09T02:33:16.472042Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the order."
}, {
"name": "placed_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was placed."
}, {
"name": "menu_item_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the menu item related to the order."
}, {
"name": "quantity",
"type": "INTEGER",
"tags": [],
"description": "The number of the item in the order."
}, {
"name": "discount_id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the discount applied to the order."
}, {
"name": "comment",
"type": "VARCHAR",
"tags": [],
"description": "The comment of the order."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:03:01.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:etl_orders.etl_orders",
"destination": "dataset:food_delivery:public.orders"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.orders",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}]
}, {
"id": "dataset:food_delivery:public.orders_7_days",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.orders_7_days"
},
"type": "DB_TABLE",
"name": "public.orders_7_days",
"physicalName": "public.orders_7_days",
"createdAt": "2021-03-09T02:33:20.794369Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "order_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the order."
}, {
"name": "placed_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was placed."
}, {
"name": "menu_id",
"type": "VARCHAR",
"tags": [],
"description": "The ID of the menu related to the order."
}, {
"name": "menu_item_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the menu item related to the order."
}, {
"name": "category_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of category related to the order."
}, {
"name": "discount_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the discount applied to the order."
}, {
"name": "city_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the city related to the order."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:08:09.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl_orders.etl_orders_7_days",
"destination": "dataset:food_delivery:public.orders_7_days"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.orders_7_days",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}]
}, {
"id": "dataset:food_delivery:public.popular_orders_day_of_week",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.popular_orders_day_of_week"
},
"type": "DB_TABLE",
"name": "public.popular_orders_day_of_week",
"physicalName": "public.popular_orders_day_of_week",
"createdAt": "2021-03-09T02:33:22.794238Z",
"updatedAt": "2022-08-04T05:02:19.384484Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "order_day_of_week",
"type": "VARCHAR",
"tags": [],
"description": "The day of week of the order."
}, {
"name": "order_placed_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was placed."
}, {
"name": "orders_placed",
"type": "INTEGER",
"tags": [],
"description": "The number of orders placed on day of week."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:18:52.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:analytics.orders_popular_day_of_week",
"destination": "dataset:food_delivery:public.popular_orders_day_of_week"
}],
"outEdges": []
}, {
"id": "dataset:food_delivery:public.restaurants",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.restaurants"
},
"type": "DB_TABLE",
"name": "public.restaurants",
"physicalName": "public.restaurants",
"createdAt": "2021-03-09T02:33:19.475385Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "id",
"type": "INTEGER",
"tags": [],
"description": "The unique ID of the restaurant."
}, {
"name": "created_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the restaurant was created."
}, {
"name": "updated_at",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the restaurant was updated."
}, {
"name": "name",
"type": "VARCHAR",
"tags": [],
"description": "The name of the restaurant."
}, {
"name": "email",
"type": "VARCHAR",
"tags": [],
"description": "The email address of the customer."
}, {
"name": "address",
"type": "VARCHAR",
"tags": [],
"description": "The address of the restaurant."
}, {
"name": "phone",
"type": "VARCHAR",
"tags": [],
"description": "The phone number of the restaurant."
}, {
"name": "city_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the city related to the restaurant."
}, {
"name": "business_hours_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the business hours related to the restaurant."
}, {
"name": "description",
"type": "TEXT",
"tags": [],
"description": "The description of the restaurant."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:06:12.190723Z",
"description": null,
"lastlifecycleState": ""
},
"inEdges": [{
"origin": "job:food_delivery:etl.etl_restaurants",
"destination": "dataset:food_delivery:public.restaurants"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.restaurants",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}]
}, {
"id": "dataset:food_delivery:public.top_delivery_times",
"type": "DATASET",
"data": {
"type": "DATASET",
"id": {
"namespace": "food_delivery",
"name": "public.top_delivery_times"
},
"type": "DB_TABLE",
"name": "public.top_delivery_times",
"physicalName": "public.top_delivery_times",
"createdAt": "2021-03-09T02:33:22.352665Z",
"updatedAt": "2022-08-04T05:18:52.190723Z",
"namespace": "food_delivery",
"sourceName": "analytics_db",
"fields": [{
"name": "order_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the order."
}, {
"name": "order_placed_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was placed."
}, {
"name": "order_dispatched_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was dispatched."
}, {
"name": "order_delivered_on",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the date/time the order was delivered."
}, {
"name": "order_delivered_time",
"type": "TIMESTAMP",
"tags": [],
"description": "An ISO-8601 timestamp representing the total time of delivery."
}, {
"name": "customer_email",
"type": "VARCHAR",
"tags": [],
"description": "The email of the customer."
}, {
"name": "restaurant_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the restaurant related to the order."
}, {
"name": "driver_id",
"type": "INTEGER",
"tags": [],
"description": "The ID of the driver related to the order."
}],
"tags": [],
"lastModifiedAt": "2022-08-04T05:15:38.190723Z",
"description": null,
"lastlifecycleState": null
},
"inEdges": [{
"origin": "job:food_delivery:analytics.delivery_times_7_days",
"destination": "dataset:food_delivery:public.top_delivery_times"
}],
"outEdges": [{
"origin": "dataset:food_delivery:public.top_delivery_times",
"destination": "job:food_delivery:analytics.orders_popular_day_of_week"
}]
}, {
"id": "job:food_delivery:analytics.delivery_times_7_days",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "analytics.delivery_times_7_days"
},
"type": "BATCH",
"name": "analytics.delivery_times_7_days",
"createdAt": "2021-07-21T05:00:12.458719Z",
"updatedAt": "2022-08-04T05:15:38.190723Z",
"namespace": "food_delivery",
"inputs": [{
"namespace": "food_delivery",
"name": "public.delivery_7_days"
}],
"outputs": [{
"namespace": "food_delivery",
"name": "public.top_delivery_times"
}, {
"namespace": "food_delivery",
"name": "public.discounts"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py",
"sql": "INSERT INTO top_delivery_times (order_id, order_placed_on, order_dispatched_on, order_delivered_on, order_delivery_time,\n customer_email, restaurant_id, driver_id)\n SELECT order_id, order_placed_on, order_delivered_on, DATEDIFF(minute, order_placed_on, order_delivered_on) AS order_delivery_time,\n customer_email, restaurant_id, driver_id\n FROM delivery_7_days\nGROUP BY restaurant_id\nORDER BY order_delivery_time DESC\n LIMIT 1;"
},
"description": "Determine weekly top delivery times by restaurant.",
"latestRun": {
"id": "3d4d4573-6419-47f7-ae0f-e0ed17784beb",
"createdAt": "2022-08-04T05:02:18.275431Z",
"updatedAt": "2022-08-04T05:15:38.190723Z",
"nominalStartTime": "2022-08-04T05:12:00Z",
"nominalEndTime": "2022-08-04T05:15:00Z",
"state": "FAILED",
"startedAt": "2022-08-04T05:12:12.190723Z",
"endedAt": "2022-08-04T05:15:38.190723Z",
"durationMs": 206000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "analytics.delivery_times_7_days",
"version": "41945ccd-36d0-3eb4-b3dd-a78fdf78aa79"
},
"inputVersions": [{
"namespace": "food_delivery",
"name": "public.delivery_7_days",
"version": "3129ce46-2ef4-3cce-8f47-6ffdee72f7a9"
}],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.top_delivery_times",
"version": "c8fc976a-2eda-353d-8e87-0f2be6b6b5ba"
}, {
"namespace": "food_delivery",
"name": "public.discounts",
"version": "0f480ffa-a812-33d1-8783-7181a82de578"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py",
"sql": "INSERT INTO top_delivery_times (order_id, order_placed_on, order_dispatched_on, order_delivered_on, order_delivery_time,\n customer_email, restaurant_id, driver_id)\n SELECT order_id, order_placed_on, order_delivered_on, DATEDIFF(minute, order_placed_on, order_delivered_on) AS order_delivery_time,\n customer_email, restaurant_id, driver_id\n FROM delivery_7_days\nGROUP BY restaurant_id\nORDER BY order_delivery_time DESC\n LIMIT 1;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:15:00Z",
"nominalStartTime": "2022-08-04T05:12:00Z"
}
}
}
},
"inEdges": [{
"origin": "dataset:food_delivery:public.delivery_7_days",
"destination": "job:food_delivery:analytics.delivery_times_7_days"
}],
"outEdges": [{
"origin": "job:food_delivery:analytics.delivery_times_7_days",
"destination": "dataset:food_delivery:public.discounts"
}, {
"origin": "job:food_delivery:analytics.delivery_times_7_days",
"destination": "dataset:food_delivery:public.top_delivery_times"
}]
}, {
"id": "job:food_delivery:analytics.orders_popular_day_of_week",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "analytics.orders_popular_day_of_week"
},
"type": "BATCH",
"name": "analytics.orders_popular_day_of_week",
"createdAt": "2021-07-21T05:00:12.665794Z",
"updatedAt": "2022-08-04T05:18:52.190723Z",
"namespace": "food_delivery",
"inputs": [{
"namespace": "food_delivery",
"name": "public.top_delivery_times"
}, {
"namespace": "food_delivery",
"name": "public.customers"
}],
"outputs": [{
"namespace": "food_delivery",
"name": "public.popular_orders_day_of_week"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py",
"sql": "INSERT INTO popular_orders_day_of_week (order_day_of_week, order_placed_on, orders_placed)\n SELECT order_day_of_week, order_placed_on, COUNT(*)\n FROM top_delivery_times;"
},
"description": "Determines the popular day of week orders are placed.",
"latestRun": {
"id": "f9eb940e-01b6-408c-bc8d-e73cea1531b7",
"createdAt": "2022-08-04T05:02:18.984244Z",
"updatedAt": "2022-08-04T05:18:52.190723Z",
"nominalStartTime": "2022-08-04T05:16:00Z",
"nominalEndTime": "2022-08-04T05:18:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:16:12.190723Z",
"endedAt": "2022-08-04T05:18:52.190723Z",
"durationMs": 160000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "analytics.orders_popular_day_of_week",
"version": "a9ab4908-971d-3989-b2ae-1f3c61a52f9e"
},
"inputVersions": [{
"namespace": "food_delivery",
"name": "public.customers",
"version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82"
}, {
"namespace": "food_delivery",
"name": "public.top_delivery_times",
"version": "16582b10-76c4-3821-baaf-c3c724f5fb52"
}],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.popular_orders_day_of_week",
"version": "72b355d1-3654-389c-b63c-2d2ac508a49a"
}, {
"namespace": "food_delivery",
"name": "public.popular_orders_day_of_week",
"version": "2f87c49c-3aee-3414-9645-09ddd13e2635"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py",
"sql": "INSERT INTO popular_orders_day_of_week (order_day_of_week, order_placed_on, orders_placed)\n SELECT order_day_of_week, order_placed_on, COUNT(*)\n FROM top_delivery_times;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:18:00Z",
"nominalStartTime": "2022-08-04T05:16:00Z"
}
}
}
},
"inEdges": [{
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:analytics.orders_popular_day_of_week"
}, {
"origin": "dataset:food_delivery:public.top_delivery_times",
"destination": "job:food_delivery:analytics.orders_popular_day_of_week"
}],
"outEdges": [{
"origin": "job:food_delivery:analytics.orders_popular_day_of_week",
"destination": "dataset:food_delivery:public.popular_orders_day_of_week"
}]
}, {
"id": "job:food_delivery:emails.email_discounts",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "emails.email_discounts"
},
"type": "BATCH",
"name": "emails.email_discounts",
"createdAt": "2021-07-21T05:00:12.563025Z",
"updatedAt": "2022-08-04T05:18:37.190723Z",
"namespace": "food_delivery",
"inputs": [{
"namespace": "food_delivery",
"name": "public.customers"
}, {
"namespace": "food_delivery",
"name": "public.discounts"
}],
"outputs": [],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py",
"sql": "SELECT * FROM discounts;"
},
"description": "Email discounts to customers that have experienced order delays.",
"latestRun": {
"id": "768ec627-2c32-40d2-b42e-1a42ada49221",
"createdAt": "2022-08-04T05:02:18.671344Z",
"updatedAt": "2022-08-04T05:18:37.190723Z",
"nominalStartTime": "2022-08-04T05:16:00Z",
"nominalEndTime": "2022-08-04T05:18:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:16:12.190723Z",
"endedAt": "2022-08-04T05:18:37.190723Z",
"durationMs": 145000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "emails.email_discounts",
"version": "84c93939-abfa-353f-9bd6-3fded9ae2e84"
},
"inputVersions": [{
"namespace": "food_delivery",
"name": "public.discounts",
"version": "93c4e393-0ed3-3aba-bd2b-787f5fcf9077"
}, {
"namespace": "food_delivery",
"name": "public.customers",
"version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82"
}],
"outputVersions": [],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py",
"sql": "SELECT * FROM discounts;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:18:00Z",
"nominalStartTime": "2022-08-04T05:16:00Z"
}
}
}
},
"inEdges": [{
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:emails.email_discounts"
}, {
"origin": "dataset:food_delivery:public.discounts",
"destination": "job:food_delivery:emails.email_discounts"
}],
"outEdges": []
}, {
"id": "job:food_delivery:etl.etl_customers",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl.etl_customers"
},
"type": "BATCH",
"name": "etl.etl_customers",
"createdAt": "2021-07-21T05:00:12.074910Z",
"updatedAt": "2022-08-04T05:07:21.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.customers"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py",
"sql": "INSERT INTO customers (id, created_at, updated_at, name, email, phone, city_id)\n SELECT id, created_at, updated_at, name, email, phone, city_id\n FROM tmp_customers;"
},
"description": "Loads newly registered customers daily.",
"latestRun": {
"id": "c06999f5-4fe0-443a-bf52-37c329019fc6",
"createdAt": "2022-08-04T05:02:15.705905Z",
"updatedAt": "2022-08-04T05:07:21.190723Z",
"nominalStartTime": "2022-08-04T05:04:00Z",
"nominalEndTime": "2022-08-04T05:07:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:04:12.190723Z",
"endedAt": "2022-08-04T05:07:21.190723Z",
"durationMs": 189000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl.etl_customers",
"version": "2fcacd14-f730-358c-b5ba-9bd3055f606b"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.customers",
"version": "be2c578f-d09a-393f-b7f2-a6c26f82d046"
}, {
"namespace": "food_delivery",
"name": "public.customers",
"version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py",
"sql": "INSERT INTO customers (id, created_at, updated_at, name, email, phone, city_id)\n SELECT id, created_at, updated_at, name, email, phone, city_id\n FROM tmp_customers;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:07:00Z",
"nominalStartTime": "2022-08-04T05:04:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl.etl_customers",
"destination": "dataset:food_delivery:public.customers"
}]
}, {
"id": "job:food_delivery:etl.etl_delivery_7_days",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl.etl_delivery_7_days"
},
"type": "BATCH",
"name": "etl.etl_delivery_7_days",
"createdAt": "2021-07-21T05:00:12.360090Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"namespace": "food_delivery",
"inputs": [{
"namespace": "food_delivery",
"name": "public.restaurants"
}, {
"namespace": "food_delivery",
"name": "public.customers"
}, {
"namespace": "food_delivery",
"name": "public.drivers"
}, {
"namespace": "food_delivery",
"name": "public.order_status"
}, {
"namespace": "food_delivery",
"name": "public.orders_7_days"
}],
"outputs": [{
"namespace": "food_delivery",
"name": "public.delivery_7_days"
}],
"location": "https://github.com/DatakinHQ/demo/blob/16adfec8fd1f37ad58e0d3c0102b3e00c4f02837/airflow/dags/etl_delivery_7_days.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/16adfec8fd1f37ad58e0d3c0102b3e00c4f02837/airflow/dags/etl_delivery_7_days.py",
"sql": "INSERT INTO delivery (order_id, order_placed_on, order_dispatched_on, order_delivered_on, customer_email,\n customer_address, discount_id, menu_id, restaurant_id, restaurant_address, menu_item_id, category_id, driver_id)\n SELECT o.order_id, o.placed_on AS order_placed_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DISPATCHED') AS order_dispatched_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DELIVERED') AS order_delivered_on,\n c.email AS customer_email, c.address AS customer_address, o.discount_id, o.menu_id, o.restaurant_id,\n r.address, o.menu_item_id, o.category_id, d.id AS driver_id\n FROM orders_7_days AS o\n INNER JOIN order_status AS os\n ON os.order_id = o.order_id\n INNER JOIN customers AS c\n ON c.id = os.customer_id\n INNER JOIN restaurants AS r\n ON r.id = os.restaurant_id\n INNER JOIN drivers AS d\n ON d.id = os.driver_id\n WHERE os.transitioned_at >= NOW() - interval '7 days';"
},
"description": "Loads new deliveries for the week.",
"latestRun": {
"id": "cda33f0a-5ee1-4be7-aa77-e553b9ff8fdc",
"createdAt": "2022-08-04T05:02:17.321081Z",
"updatedAt": "2022-08-04T05:10:50.190723Z",
"nominalStartTime": "2022-08-04T05:08:00Z",
"nominalEndTime": "2022-08-04T05:10:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:08:12.190723Z",
"endedAt": "2022-08-04T05:10:50.190723Z",
"durationMs": 158000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl.etl_delivery_7_days",
"version": "bc6c294b-b0eb-3160-a06d-1ff9ba3a4e1c"
},
"inputVersions": [{
"namespace": "food_delivery",
"name": "public.orders_7_days",
"version": "3dc3a809-8bdc-3c50-809d-fa24e5d48455"
}, {
"namespace": "food_delivery",
"name": "public.order_status",
"version": "1261ea1c-cb68-3ad1-b208-6c4c65714b8e"
}, {
"namespace": "food_delivery",
"name": "public.drivers",
"version": "9db6f800-ffcc-3eac-9c7a-86cd9d702c27"
}, {
"namespace": "food_delivery",
"name": "public.restaurants",
"version": "fcb2cece-ccd6-3fe0-a1c7-52bd0e320040"
}, {
"namespace": "food_delivery",
"name": "public.customers",
"version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82"
}],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.delivery_7_days",
"version": "66fcf19a-e24a-3520-9d24-d78d75572066"
}, {
"namespace": "food_delivery",
"name": "public.delivery_7_days",
"version": "3129ce46-2ef4-3cce-8f47-6ffdee72f7a9"
}],
"context": {
"sql": "INSERT INTO delivery (order_id, order_placed_on, order_dispatched_on, order_delivered_on, customer_email,\n customer_address, discount_id, menu_id, restaurant_id, restaurant_address, menu_item_id, category_id, driver_id)\n SELECT o.order_id, o.placed_on AS order_placed_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DISPATCHED') AS order_dispatched_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DELIVERED') AS order_delivered_on,\n c.email AS customer_email, c.address AS customer_address, o.discount_id, o.menu_id, o.restaurant_id,\n r.address, o.menu_item_id, o.category_id, d.id AS driver_id\n FROM orders_7_days AS o\n INNER JOIN order_status AS os\n ON os.order_id = o.order_id\n INNER JOIN customers AS c\n ON c.id = os.customer_id\n INNER JOIN restaurants AS r\n ON r.id = os.restaurant_id\n INNER JOIN drivers AS d\n ON d.id = os.driver_id\n WHERE os.transitioned_at >= NOW() - interval '7 days';"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:10:00Z",
"nominalStartTime": "2022-08-04T05:08:00Z"
}
}
}
},
"inEdges": [{
"origin": "dataset:food_delivery:public.customers",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}, {
"origin": "dataset:food_delivery:public.drivers",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}, {
"origin": "dataset:food_delivery:public.order_status",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}, {
"origin": "dataset:food_delivery:public.orders_7_days",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}, {
"origin": "dataset:food_delivery:public.restaurants",
"destination": "job:food_delivery:etl.etl_delivery_7_days"
}],
"outEdges": [{
"origin": "job:food_delivery:etl.etl_delivery_7_days",
"destination": "dataset:food_delivery:public.delivery_7_days"
}]
}, {
"id": "job:food_delivery:etl.etl_drivers",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl.etl_drivers"
},
"type": "BATCH",
"name": "etl.etl_drivers",
"createdAt": "2021-07-21T05:00:12.175032Z",
"updatedAt": "2022-08-04T05:06:40.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.drivers"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py",
"sql": "INSERT INTO drivers (id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate)\n SELECT id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate\n FROM tmp_drivers;"
},
"description": "Loads newly registered drivers daily.",
"latestRun": {
"id": "af9b88b0-af2e-44b0-a076-a5b21f54faba",
"createdAt": "2022-08-04T05:02:16.321336Z",
"updatedAt": "2022-08-04T05:06:40.190723Z",
"nominalStartTime": "2022-08-04T05:04:00Z",
"nominalEndTime": "2022-08-04T05:06:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:04:12.190723Z",
"endedAt": "2022-08-04T05:06:40.190723Z",
"durationMs": 148000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl.etl_drivers",
"version": "c552ff83-87a2-386f-8c5d-c62c66773c06"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.drivers",
"version": "7fbda958-e49f-3852-80a4-1878b8d29aeb"
}, {
"namespace": "food_delivery",
"name": "public.drivers",
"version": "9db6f800-ffcc-3eac-9c7a-86cd9d702c27"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py",
"sql": "INSERT INTO drivers (id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate)\n SELECT id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate\n FROM tmp_drivers;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:06:00Z",
"nominalStartTime": "2022-08-04T05:04:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl.etl_drivers",
"destination": "dataset:food_delivery:public.drivers"
}]
}, {
"id": "job:food_delivery:etl.etl_order_status",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl.etl_order_status"
},
"type": "BATCH",
"name": "etl.etl_order_status",
"createdAt": "2021-07-21T05:00:12.157147Z",
"updatedAt": "2022-08-04T05:07:05.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.order_status"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py",
"sql": "INSERT INTO order_status (id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id)\n SELECT id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id\n FROM tmp_order_status;"
},
"description": "Loads order statues updates daily.",
"latestRun": {
"id": "a847b17b-7772-4606-b0d3-f0371a4358eb",
"createdAt": "2022-08-04T05:02:16.018871Z",
"updatedAt": "2022-08-04T05:07:05.190723Z",
"nominalStartTime": "2022-08-04T05:04:00Z",
"nominalEndTime": "2022-08-04T05:07:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:04:12.190723Z",
"endedAt": "2022-08-04T05:07:05.190723Z",
"durationMs": 173000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl.etl_order_status",
"version": "2e04b236-747d-3ba2-a982-1acadda139d1"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.order_status",
"version": "2298c453-1a77-3daf-baa7-77057a89d1c1"
}, {
"namespace": "food_delivery",
"name": "public.order_status",
"version": "1261ea1c-cb68-3ad1-b208-6c4c65714b8e"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py",
"sql": "INSERT INTO order_status (id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id)\n SELECT id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id\n FROM tmp_order_status;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:07:00Z",
"nominalStartTime": "2022-08-04T05:04:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl.etl_order_status",
"destination": "dataset:food_delivery:public.order_status"
}]
}, {
"id": "job:food_delivery:etl.etl_restaurants",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl.etl_restaurants"
},
"type": "BATCH",
"name": "etl.etl_restaurants",
"createdAt": "2021-07-21T05:00:12.052261Z",
"updatedAt": "2022-08-04T05:06:12.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.restaurants"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py",
"sql": "INSERT INTO restaurants (id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description)\n SELECT id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description\n FROM tmp_restaurants;"
},
"description": "Loads newly registered restaurants daily.",
"latestRun": {
"id": "e36d9152-9c01-4901-ad51-0d54e176ab00",
"createdAt": "2022-08-04T05:02:15.375517Z",
"updatedAt": "2022-08-04T05:06:12.190723Z",
"nominalStartTime": "2022-08-04T05:04:00Z",
"nominalEndTime": "2022-08-04T05:06:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:04:12.190723Z",
"endedAt": "2022-08-04T05:06:12.190723Z",
"durationMs": 120000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl.etl_restaurants",
"version": "d51962aa-cf5f-3e0e-b240-fc1e5dbf4879"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.restaurants",
"version": "3ba477ce-4016-3852-8d38-25bf68cdc1cd"
}, {
"namespace": "food_delivery",
"name": "public.restaurants",
"version": "fcb2cece-ccd6-3fe0-a1c7-52bd0e320040"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py",
"sql": "INSERT INTO restaurants (id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description)\n SELECT id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description\n FROM tmp_restaurants;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:06:00Z",
"nominalStartTime": "2022-08-04T05:04:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl.etl_restaurants",
"destination": "dataset:food_delivery:public.restaurants"
}]
}, {
"id": "job:food_delivery:etl_orders.etl_categories",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl_orders.etl_categories"
},
"type": "BATCH",
"name": "etl_orders.etl_categories",
"createdAt": "2021-07-21T05:00:11.870558Z",
"updatedAt": "2022-08-04T05:03:09.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.categories"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py",
"sql": "INSERT INTO categories (id, name, menu_id, description)\n SELECT id, name, menu_id, description\n FROM tmp_categories;"
},
"description": "Loads newly added menus categories daily.",
"latestRun": {
"id": "24a62ce4-4239-4aa4-b1c2-d1e3950349cc",
"createdAt": "2022-08-04T05:02:14.592636Z",
"updatedAt": "2022-08-04T05:03:09.190723Z",
"nominalStartTime": "2022-08-04T05:00:00Z",
"nominalEndTime": "2022-08-04T05:03:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:00:12.190723Z",
"endedAt": "2022-08-04T05:03:09.190723Z",
"durationMs": 177000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl_orders.etl_categories",
"version": "4c409e02-f046-34e0-9336-a3f71c2af478"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.categories",
"version": "aba09e19-b23e-3601-987d-590965c6aae3"
}, {
"namespace": "food_delivery",
"name": "public.categories",
"version": "d4ea4286-d8f3-3640-bad0-29afa61eb52d"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py",
"sql": "INSERT INTO categories (id, name, menu_id, description)\n SELECT id, name, menu_id, description\n FROM tmp_categories;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:03:00Z",
"nominalStartTime": "2022-08-04T05:00:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl_orders.etl_categories",
"destination": "dataset:food_delivery:public.categories"
}]
}, {
"id": "job:food_delivery:etl_orders.etl_menu_items",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl_orders.etl_menu_items"
},
"type": "BATCH",
"name": "etl_orders.etl_menu_items",
"createdAt": "2021-07-21T05:00:11.962073Z",
"updatedAt": "2022-08-04T05:03:24.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.menu_items"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py",
"sql": "INSERT INTO menu_items (id, name, price, category_id, description)\n SELECT id, name, price, category_id, description\n FROM tmp_menu_items;"
},
"description": "Loads newly added restaurant menu items daily.",
"latestRun": {
"id": "6ca9eafe-ebbd-4edb-803e-f4c6772528db",
"createdAt": "2022-08-04T05:02:15.086486Z",
"updatedAt": "2022-08-04T05:03:24.190723Z",
"nominalStartTime": "2022-08-04T05:00:00Z",
"nominalEndTime": "2022-08-04T05:03:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:00:12.190723Z",
"endedAt": "2022-08-04T05:03:24.190723Z",
"durationMs": 192000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl_orders.etl_menu_items",
"version": "b7bbdfa9-32e0-39c5-93d1-83a4d406d4b6"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.menu_items",
"version": "80a5d913-ec71-3aeb-bb61-a14763dad31d"
}, {
"namespace": "food_delivery",
"name": "public.menu_items",
"version": "ee00a374-4a87-3606-8447-b4000aa5090b"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py",
"sql": "INSERT INTO menu_items (id, name, price, category_id, description)\n SELECT id, name, price, category_id, description\n FROM tmp_menu_items;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:03:00Z",
"nominalStartTime": "2022-08-04T05:00:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl_orders.etl_menu_items",
"destination": "dataset:food_delivery:public.menu_items"
}]
}, {
"id": "job:food_delivery:etl_orders.etl_menus",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl_orders.etl_menus"
},
"type": "BATCH",
"name": "etl_orders.etl_menus",
"createdAt": "2021-07-21T05:00:11.849068Z",
"updatedAt": "2022-08-04T05:03:06.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.menus"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py",
"sql": "INSERT INTO menus (id, name, restaurant_id, description)\n SELECT id, name, restaurant_id, description\n FROM tmp_menus;"
},
"description": "Loads newly added restaurant menus daily.",
"latestRun": {
"id": "418fe5ef-3205-42c9-bfc6-11d4c89f71d0",
"createdAt": "2022-08-04T05:02:14.310639Z",
"updatedAt": "2022-08-04T05:03:06.190723Z",
"nominalStartTime": "2022-08-04T05:00:00Z",
"nominalEndTime": "2022-08-04T05:03:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:00:12.190723Z",
"endedAt": "2022-08-04T05:03:06.190723Z",
"durationMs": 174000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl_orders.etl_menus",
"version": "f71aea64-be54-30b9-a370-e43fa6c5d518"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.menus",
"version": "a25c18e1-737a-3121-b9d8-e50fbd5ccf4a"
}, {
"namespace": "food_delivery",
"name": "public.menus",
"version": "5b4890a4-863c-3a87-8f88-99d64f0d38dc"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py",
"sql": "INSERT INTO menus (id, name, restaurant_id, description)\n SELECT id, name, restaurant_id, description\n FROM tmp_menus;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:03:00Z",
"nominalStartTime": "2022-08-04T05:00:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl_orders.etl_menus",
"destination": "dataset:food_delivery:public.menus"
}]
}, {
"id": "job:food_delivery:etl_orders.etl_orders",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl_orders.etl_orders"
},
"type": "BATCH",
"name": "etl_orders.etl_orders",
"createdAt": "2021-07-21T05:00:11.754754Z",
"updatedAt": "2022-08-04T05:03:01.190723Z",
"namespace": "food_delivery",
"inputs": [],
"outputs": [{
"namespace": "food_delivery",
"name": "public.orders"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py",
"sql": "INSERT INTO orders (id, placed_on, menu_item_id, quantity, discount_id, comment)\n SELECT id, placed_on, menu_item_id, quantity, discount_id, comment\n FROM tmp_orders;"
},
"description": "Loads newly placed orders weekly.",
"latestRun": {
"id": "9c498876-50e0-4767-b57c-7eb898f48bd2",
"createdAt": "2022-08-04T05:02:14.012477Z",
"updatedAt": "2022-08-04T05:03:01.190723Z",
"nominalStartTime": "2022-08-04T05:00:00Z",
"nominalEndTime": "2022-08-04T05:03:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:00:12.190723Z",
"endedAt": "2022-08-04T05:03:01.190723Z",
"durationMs": 169000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl_orders.etl_orders",
"version": "5cd12692-13b8-3409-a80a-d670c77ba9af"
},
"inputVersions": [],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.orders",
"version": "acbe5e24-fcfc-3323-abed-60e12f6016f4"
}, {
"namespace": "food_delivery",
"name": "public.orders",
"version": "42d5a311-e3df-3ab6-bc8b-baed3a5910b0"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py",
"sql": "INSERT INTO orders (id, placed_on, menu_item_id, quantity, discount_id, comment)\n SELECT id, placed_on, menu_item_id, quantity, discount_id, comment\n FROM tmp_orders;"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:03:00Z",
"nominalStartTime": "2022-08-04T05:00:00Z"
}
}
}
},
"inEdges": [],
"outEdges": [{
"origin": "job:food_delivery:etl_orders.etl_orders",
"destination": "dataset:food_delivery:public.orders"
}]
}, {
"id": "job:food_delivery:etl_orders.etl_orders_7_days",
"type": "JOB",
"data": {
"type": "JOB",
"id": {
"namespace": "food_delivery",
"name": "etl_orders.etl_orders_7_days"
},
"type": "BATCH",
"name": "etl_orders.etl_orders_7_days",
"createdAt": "2021-07-21T05:00:12.264069Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"namespace": "food_delivery",
"inputs": [{
"namespace": "food_delivery",
"name": "public.categories"
}, {
"namespace": "food_delivery",
"name": "public.menu_items"
}, {
"namespace": "food_delivery",
"name": "public.orders"
}, {
"namespace": "food_delivery",
"name": "public.menus"
}],
"outputs": [{
"namespace": "food_delivery",
"name": "public.orders_7_days"
}],
"location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py",
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py",
"sql": "INSERT INTO orders_7_days (order_id, placed_on, discount_id, menu_id, restaurant_id, menu_item_id, category_id)\n SELECT o.id AS order_id, o.placed_on, o.discount_id, m.id AS menu_id, m.restaurant_id, mi.id AS menu_item_id, c.id AS category_id\n FROM orders AS o\n INNER JOIN menu_items AS mi\n ON menu_items.id = o.menu_item_id\n INNER JOIN categories AS c\n ON c.id = mi.category_id\n INNER JOIN menu AS m\n ON m.id = c.menu_id\n WHERE o.placed_on >= NOW() - interval '7 days';"
},
"description": "Loads newly placed orders weekly.",
"latestRun": {
"id": "ad6cf477-b233-4305-9bc3-4cc48fe08047",
"createdAt": "2022-08-04T05:02:16.660488Z",
"updatedAt": "2022-08-04T05:08:09.190723Z",
"nominalStartTime": "2022-08-04T05:04:00Z",
"nominalEndTime": "2022-08-04T05:08:00Z",
"state": "COMPLETED",
"startedAt": "2022-08-04T05:04:12.190723Z",
"endedAt": "2022-08-04T05:08:09.190723Z",
"durationMs": 237000,
"args": {},
"jobVersion": {
"namespace": "food_delivery",
"name": "etl_orders.etl_orders_7_days",
"version": "e1cfd2ef-8197-34e4-b361-8ffc012cff81"
},
"inputVersions": [{
"namespace": "food_delivery",
"name": "public.orders",
"version": "42d5a311-e3df-3ab6-bc8b-baed3a5910b0"
}, {
"namespace": "food_delivery",
"name": "public.menu_items",
"version": "ee00a374-4a87-3606-8447-b4000aa5090b"
}, {
"namespace": "food_delivery",
"name": "public.menus",
"version": "5b4890a4-863c-3a87-8f88-99d64f0d38dc"
}, {
"namespace": "food_delivery",
"name": "public.categories",
"version": "d4ea4286-d8f3-3640-bad0-29afa61eb52d"
}],
"outputVersions": [{
"namespace": "food_delivery",
"name": "public.orders_7_days",
"version": "3c7f6da4-e79a-3eeb-83c6-dcffe8b9ced9"
}, {
"namespace": "food_delivery",
"name": "public.orders_7_days",
"version": "3dc3a809-8bdc-3c50-809d-fa24e5d48455"
}],
"context": {
"job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py",
"sql": "INSERT INTO orders_7_days (order_id, placed_on, discount_id, menu_id, restaurant_id, menu_item_id, category_id)\n SELECT o.id AS order_id, o.placed_on, o.discount_id, m.id AS menu_id, m.restaurant_id, mi.id AS menu_item_id, c.id AS category_id\n FROM orders AS o\n INNER JOIN menu_items AS mi\n ON menu_items.id = o.menu_item_id\n INNER JOIN categories AS c\n ON c.id = mi.category_id\n INNER JOIN menu AS m\n ON m.id = c.menu_id\n WHERE o.placed_on >= NOW() - interval '7 days';"
},
"facets": {
"nominalTime": {
"_producer": "com.datakin.cli.SeedCommand",
"_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet",
"nominalEndTime": "2022-08-04T05:08:00Z",
"nominalStartTime": "2022-08-04T05:04:00Z"
}
}
}
},
"inEdges": [{
"origin": "dataset:food_delivery:public.categories",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}, {
"origin": "dataset:food_delivery:public.menu_items",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}, {
"origin": "dataset:food_delivery:public.menus",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}, {
"origin": "dataset:food_delivery:public.orders",
"destination": "job:food_delivery:etl_orders.etl_orders_7_days"
}],
"outEdges": [{
"origin": "job:food_delivery:etl_orders.etl_orders_7_days",
"destination": "dataset:food_delivery:public.orders_7_days"
}]
}]
}
This could be extended to overlay the column level lineage. For example:
GET /lineage?nodeId=dataset:food_delivery:public.delivery_7_days&column=a
Which would add column level lineage information.
We limit the scope of this to merging proposal. Rest of work will be split in separate issues under the epic: https://github.com/OpenLineage/OpenLineage/issues/1089
