Pull metadata from API Discovery Tool

[antidipyramid]

https://www.census.gov/data/developers/updates/new-discovery-tool.html

[antidipyramid]

@fgregg I made some progress using a metaclass to programmatically pull in each client's supported years, default year, and enpoint urls from the discovery tool:

import requests
from functools import lru_cache


@lru_cache
def fetch_metadata():
    # Fetches bulk metadata from the US Census' Discovery Tool API
    r = requests.get("https://api.census.gov/data.json")
    return r.json()


class ClientMeta(type):
    def __init__(cls, name, bases, dct):
        metadata = fetch_metadata()

        if not hasattr(cls, "dataset"):
            return super().__init__(name, bases, dct)

        # Initialize years attribute
        setattr(cls, "years", set())

        # e.g. ACS5Client.dataset is "acs/acs5" which corresponds to a metadata
        # object with a dataset attribute of ["acs", "acs5"]
        dataset = cls.dataset.split("/")

        for m in metadata["dataset"]:
            if "c_vintage" not in m:
                continue

            if m["c_dataset"] == dataset:
                cls.years.add(m["c_vintage"])

                # Set API URL attributes
                setattr(cls, "definitions_url", m["c_variablesLink"])
                setattr(cls, "groups_url", m["c_groupsLink"])

                setattr(cls, "default_year", max(cls.years))

        return super().__init__(name, bases, dct)

Then we can just define Client like:

class Client(metaclass=ClientMeta):
    def __init__(self, key, year=None, session=None, retries=3):
        ...

Seems like this runs once on package import for each client in core.py.

What do you think about this approach, @fgregg?

[fgregg]

love it.