Y42 simplifies data integration from external APIs using Python. Our platform handles the infrastructure and eliminates the need for boilerplate code for loading data into the Data Warehouse (DWH).
Your main task is writing Python logic to fetch data into a DataFrame, with each DataFrame representing a unique source table. These can then be modeled and processed downstream like any other source. Furthermore, Python ingest assets are subjected to version control and comply with the Virtual Data Builds mechanism. This ensures consistency and reliability in pipelines that utilize Python Ingest assets as sources.
from y42.v1.decorators import data_loader
import os
import requests
import json
import pandas as pd
@data_loader
def pizza_status(context) -> pd.DataFrame:
url = "https://database.supabase.co/rest/v1/orders?select=status"
api_key = context.secrets.get("SUPABASE_PIZZA_POSTGRES")
headers = {
'apikey': api_key,
'Authorization': 'Bearer ' + api_key
}
response = requests.get(url, headers=headers)
data = response.json()
df = pd.DataFrame(data)
return df