🔍 Data Discovery & Ingestion

def get_new_business_licenses(client: SodaClient, days_back: int = 90):
    """Fetch new business licenses with daily aggregation"""

    # Calculate date threshold
    from datetime import datetime, timedelta
    threshold_date = (datetime.now() - timedelta(days=days_back)).strftime('%Y-%m-%d')

    # SODA query parameters for server-side aggregation
    params = {
        '$select': 'community_area_name,community_area,license_description,' +
                   'date_trunc_ymd(license_term_start_date) AS day,count(1) AS n',
        '$where': f"application_type='ISSUE' AND license_term_start_date >= '{threshold_date}'",
        '$group': 'community_area_name,community_area,license_description,day',
        '$order': 'day',
        '$limit': 10000  # Safety limit
    }

    return client.query('r5zr-chrr', params)

def validate_business_license_data(data: list) -> bool:
    """Validate business license data quality"""

    if not data:
        return False

    # Check required fields
    required_fields = ['community_area_name', 'community_area', 'day', 'n']
    for record in data:
        if not all(field in record for field in required_fields):
            return False

    # Validate numeric values
    for record in data:
        try:
            count = int(record['n'])
            if count < 0:
                return False
        except (ValueError, TypeError):
            return False

    return True

def log_data_quality_metrics(data: list, source: str):
    """Log data quality metrics for monitoring"""
    total_records = len(data)
    unique_areas = len(set(record['community_area'] for record in data))
    unique_dates = len(set(record['day'] for record in data))

    print(f"{source} Quality Metrics:")
    print(f"   Total Records: {total_records}")
    print(f"   Unique Areas: {unique_areas}")
    print(f"   Unique Dates: {unique_dates}")
    print(f"   Date Range: {min(record['day'] for record in data)} to {max(record['day'] for record in data)}")