dbt-coverage icon indicating copy to clipboard operation
dbt-coverage copied to clipboard

Parse columns that have quotes correctly in overall count

Open miguelasse opened this issue 1 year ago • 0 comments

I don't believe columns that currently have quotes in them are being parsed correctly - I think they should show as having tests but they're not.

Here's an example from our project, it shows 3/27 as having tests, where I actually know that all 27 columns have tests, but the column themselves have spaces and are quoted in the .yml file:

(dev) ➜  analytics-env git:(miguel_dev) ✗ dbt-coverage compute test --cov-report coverage-doc.json --model-path-filter models/2-mart/3-staff
Coverage report (test)
=====================================================================
sage_prod_staff.mrt_staff_insights                     3/27     11.1%
=====================================================================

Here is the column from our .yml file for that model MRT_STAFF_INSIGHTS:

      - name: '"Top Performer"'
        description: "Boolean indicator of whether the staff member is considered a top performer."
        data_type: BOOLEAN
        data_tests:
          - not_null
          - is_boolean

Here is the field in catalog.json:

                "Top Performer": {
                    "type": "BOOLEAN",
                    "index": 5,
                    "name": "Top Performer",
                    "comment": null
                },

Here is the field in manifest.json:

                "\"Top Performer\"": {
                    "name": "\"Top Performer\"",
                    "description": "Boolean indicator of whether the staff member is considered a top performer.",
                    "meta": {},
                    "data_type": "BOOLEAN",
                    "constraints": [],
                    "quote": null,
                    "tags": []
                },

Here is the actual compiled test metadata in manifest.json:

            "compiled_path": "target/compiled/analytics/models/2-mart/3-staff/schema_tests.yml/not_null_MRT_STAFF_INSIGHTS__Top_Performer_.sql",
            "compiled": true,
            "compiled_code": "\n    \n    \n\n\n\nselect \"Top Performer\"\nfrom ANALYTICS_MART.SAGE_PROD_STAFF.MRT_STAFF_INSIGHTS\nwhere \"Top Performer\" is null\n\n\n",
            "extra_ctes_injected": true,
            "extra_ctes": [],
            "contract": {
                "enforced": false,
                "alias_types": true,
                "checksum": null
            },
            "column_name": "\"Top Performer\"",
            "file_key_name": "models.MRT_STAFF_INSIGHTS",
            "attached_node": "model.analytics.MRT_STAFF_INSIGHTS",
            "test_metadata": {
                "name": "not_null",
                "kwargs": {
                    "column_name": "\"Top Performer\"",
                    "model": "{{ get_where_subquery(ref('MRT_STAFF_INSIGHTS')) }}"
                },
                "namespace": null
            }
        },
        "test.analytics.is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_.e5e3589be6": {
            "database": "AURORA_POSTGRES_SAGE_PROD",
            "schema": "SAGE_PROD_dbt_test__audit",
            "name": "is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_",
            "resource_type": "test",
            "package_name": "analytics",
            "path": "is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_.sql",
            "original_file_path": "models/2-mart/3-staff/schema_tests.yml",
            "unique_id": "test.analytics.is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_.e5e3589be6",
            "fqn": [
                "analytics",
                "2-mart",
                "3-staff",
                "is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_"
            ],
            "alias": "is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_",
            "checksum": {
                "name": "none",
                "checksum": ""
            },
            "config": {
                "enabled": true,
                "alias": null,
                "schema": "dbt_test__audit",
                "database": null,
                "tags": [],
                "meta": {},
                "group": null,
                "materialized": "test",
                "severity": "ERROR",
                "store_failures": null,
                "store_failures_as": null,
                "where": null,
                "limit": null,
                "fail_calc": "count(*)",
                "warn_if": "!= 0",
                "error_if": "!= 0"
            },
            "tags": [],
            "description": "",
            "columns": {},
            "meta": {},
            "group": null,
            "docs": {
                "show": true,
                "node_color": null
            },
            "patch_path": null,
            "build_path": null,
            "unrendered_config": {},
            "created_at": 1722451501.6559052,
            "relation_name": null,
            "raw_code": "{{ test_is_boolean(**_dbt_generic_test_kwargs) }}",
            "language": "sql",
            "refs": [
                {
                    "name": "MRT_STAFF_INSIGHTS",
                    "package": null,
                    "version": null
                }
            ],
            "sources": [],
            "metrics": [],
            "depends_on": {
                "macros": [
                    "macro.analytics.test_is_boolean",
                    "macro.dbt.get_where_subquery"
                ],
                "nodes": [
                    "model.analytics.MRT_STAFF_INSIGHTS"
                ]
            },
            "compiled_path": "target/compiled/analytics/models/2-mart/3-staff/schema_tests.yml/is_boolean_MRT_STAFF_INSIGHTS__Top_Performer_.sql",
            "compiled": true,
            "compiled_code": "\n\n    select 1 where false  -- Always pass if the column doesn't exist\n\n",
            "extra_ctes_injected": true,
            "extra_ctes": [],
            "contract": {
                "enforced": false,
                "alias_types": true,
                "checksum": null
            },
            "column_name": "\"Top Performer\"",
            "file_key_name": "models.MRT_STAFF_INSIGHTS",
            "attached_node": "model.analytics.MRT_STAFF_INSIGHTS",
            "test_metadata": {
                "name": "is_boolean",
                "kwargs": {
                    "column_name": "\"Top Performer\"",
                    "model": "{{ get_where_subquery(ref('MRT_STAFF_INSIGHTS')) }}"
                },
                "namespace": null
            }
        },

miguelasse avatar Jul 31 '24 19:07 miguelasse