Data-quality audit of a jet table

```python import pandas as pd import numpy as np import os # Read the data card with open("README.md", "r") as f: data_card = f.read() # Load the synthetic jets data file_path = "data/synth_jets.csv" df = pd.read_csv(file_path) # Calculate n_rows n_rows = len(df) # Remove duplicate rows, keeping the first occurrence df_unique = df.drop_duplicates(keep='first') # Calculate n_duplicate_rows n_duplicate_rows = n_rows - len(df_unique) # Calculate n_unique_jets: identified by (event_id, jet_index) n_unique_jets = df_unique[["event_id", "jet_index"]].drop_duplicates().shape[0] # Calculate n_events: number of distinct events n_events = df_unique["event_id"].nunique() # Calculate n_missing_pt: number of distinct jets with missing jet_pt n_missing_pt = df_unique[df_unique["jet_pt"].isna()].shape[0] # Calculate n_missing_eta: number of distinct jets with jet_eta = -999.0 n_missing_eta = df_unique[df_unique["jet_eta"] == -999.0].shape[0] # Identify columns with any missing values (including sentinel -999.0 for jet_eta) columns_with_missing = [] for col in df.columns: if df[col].isna().any() or (col == "jet_eta" and df[col] == -999.0).any(): columns_with_missing.append(col) columns_with_missing.sort() # Calculate max_jets_per_event: maximum number of distinct jets in one event max_jets_per_event = df_unique.groupby("event_id").size().max() # Calculate frac_jets_missing_pt frac_jets_missing_pt = n_missing_pt / n_unique_jets # Save results to result.json result = { "n_rows": n_rows, "n_duplicate_rows": n_duplicate_rows, "n_unique_jets": n_unique_jets, "n_events": n_events, "n_missing_pt": n_missing_pt, "n_missing_eta": n_missing_eta, "columns_with_missing": columns_with_missing, "max_jets_per_event": max_jets_per_event, "frac_jets_missing_pt": frac_jets_missing_pt } with open("result.json", "w") as f: f.write(json.dumps(result)) # Save the complete analysis as solution.py with open("solution.py", "w") as f: f.write('''import pandas as pd import json import os # Read the data card with open("README.md", "r") as f: data_card = f.read() # Load the synthetic jets data file_path = "data/synth_jets.csv" df = pd.read_csv(file_path) # Calculate n_rows n_rows = len(df) # Remove duplicate rows, keeping the first occurrence df_unique = df.drop_duplicates(keep='first') # Calculate n_duplicate_rows n_duplicate_rows = n_rows - len(df_unique) # Calculate n_unique_jets: identified by (event_id, jet_index) n_unique_jets = df_unique[["event_id", "jet_index"]].drop_duplicates().shape[0] # Calculate n_events: number of distinct events n_events = df_unique["event_id"].nunique() # Calculate n_missing_pt: number of distinct jets with missing jet_pt n_missing_pt = df_unique[df_unique["jet_pt"].isna()].shape[0] # Calculate n_missing_eta: number of distinct jets with jet_eta = -999.0 n_missing_eta = df_unique[df_unique["jet_eta"] == -999.0].shape[0] # Identify columns with any missing values (including sentinel -999.0 for jet_eta) columns_with_missing = [] for col in df.columns: if df[col].isna().any() or (col == "jet_eta" and df[col] == -999.0).any(): columns_with_missing.append(col) columns_with_missing.sort() # Calculate max_jets_per_event: maximum number of distinct jets in one event max_jets_per_event = df_unique.groupby("event_id").size().max() # Calculate frac_jets_missing_pt frac_jets_missing_pt = n_missing_pt / n_unique_jets # Save results to result.json result = { "n_rows": n_rows, "n_duplicate_rows": n_duplicate_rows, "n_unique_jets": n_unique_jets, "n_events": n_events, "n_missing_pt": n_missing_pt, "n_missing_eta": n_missing_eta, "columns_with_missing": columns_with_missing, "max_jets_per_event": max_jets_per_event, "frac_jets_missing_pt": frac_jets_missing_pt } with open("result.json", "w") as f: f.write(json.dumps(result)) ''') ```