"""Make the demo datasets: a customer-churn training file and a later "test" file, with known problems planted in them. Uses only the standard library and a fixed seed, so the files (and the expected findings in demo_truth.json) are reproducible. python tests/make_demo.py # writes data/demo_train.csv, data/demo_test.csv, data/demo_truth.json """ import csv import json import os import random import sys HERE = os.path.dirname(os.path.abspath(__file__)) OUT = os.path.join(HERE, "..", "data") COUNTRIES = ["US", "UK", "Canada", "Germany", "France"] COUNTRY_P = [0.45, 0.2, 0.15, 0.1, 0.1] PLANS = ["Basic", "Pro", "Premium"] PLAN_PRICE = {"Basic": 19.99, "Pro": 49.99, "Premium": 99.0, "Enterprise": 249.0} REASONS = ["Too expensive", "Switched to competitor", "Missing features", "Poor support", "No longer needed"] def pick(rng, items, probs): r, acc = rng.random(), 0.0 for item, p in zip(items, probs): acc += p if r < acc: return item return items[-1] def make_customer(rng, i, test=False): age = int(min(80, max(18, rng.gauss(44 if test else 38, 11)))) country = pick(rng, COUNTRIES, COUNTRY_P) plan = pick(rng, PLANS, [0.5, 0.35, 0.15]) if test and rng.random() < 0.12: plan = "Enterprise" # a plan that does not exist in the training file tenure = max(1, int(rng.expovariate(1 / 18))) tickets = min(15, int(rng.expovariate(1 / 1.6))) last_login = int(rng.expovariate(1 / (6 if rng.random() < 0.8 else 40))) income = int(rng.lognormvariate(11.0, 0.35)) # churn depends (honestly) on tenure, tickets and last login logit = -1.3 - 0.035 * tenure + 0.35 * tickets + 0.03 * last_login + (0.4 if plan == "Basic" else 0) churned = 1 if rng.random() < 1 / (1 + 2.718281828 ** (-logit)) else 0 year = rng.choice([2022, 2023, 2024]) month, day = rng.randint(1, 12), rng.randint(1, 28) return { "customer_id": f"C-{i:05d}", "signup_date": f"{year}-{month:02d}-{day:02d}", "age": age, "country": country, "plan": plan, "monthly_spend": PLAN_PRICE[plan] * (1 + 0.1 * rng.random()), "tenure_months": tenure, "support_tickets": tickets, "last_login_days": last_login, "income": income, "email": f"user{i}@{rng.choice(['gmail.com', 'yahoo.com', 'outlook.com'])}", "account_status": "active", "referral_code": f"R{rng.randint(1000, 9999)}" if rng.random() < 0.12 else "", "cancellation_reason": rng.choice(REASONS) if churned else "", "churned": churned, "_month": month, "_day": day, "_year": year, } COLUMNS = ["customer_id", "signup_date", "age", "country", "plan", "monthly_spend", "tenure_months", "support_tickets", "last_login_days", "income", "email", "account_status", "referral_code", "cancellation_reason", "churned"] def main(): rng = random.Random(7) truth = {"train": {}, "test": {}} # ---------------- training file ---------------- n = 2000 rows = [make_customer(rng, i + 1) for i in range(n)] # 1. churned customers from a second export were appended at the end (rows sorted by target) late = [make_customer(rng, n + i + 1) for i in range(1400)] late = [r for r in late if r["churned"] == 1][:350] rows += late truth["train"]["order_leak"] = "churned" for idx, r in enumerate(rows): # 2. country spelled several ways if r["country"] == "US": r["country"] = pick(rng, ["US", "USA", "United States", "us", "US "], [0.7, 0.12, 0.08, 0.05, 0.05]) elif r["country"] == "Canada" and rng.random() < 0.15: r["country"] = "canada" # 3. a typo in plan names if r["plan"] == "Premium" and rng.random() < 0.06: r["plan"] = "Premum" # 4. money stored as text: "$49.99" and "$1,049.00" r["monthly_spend"] = "${:,.2f}".format(r["monthly_spend"]) # 5. disguised missing: -1 for unknown age, "N/A" / "unknown" for country u = rng.random() if u < 0.04: r["age"] = -1 elif u < 0.07: r["age"] = "" if rng.random() < 0.02: r["country"] = rng.choice(["N/A", "unknown"]) # 6. income: placeholder 999999 and a couple of typos with extra zeros v = rng.random() if v < 0.015: r["income"] = 999999 elif v < 0.019: r["income"] = r["income"] * 100 # 7. mixed date formats: some rows from the old system are MM/DD/YYYY if rng.random() < 0.25: r["signup_date"] = f"{r['_month']:02d}/{r['_day']:02d}/{r['_year']}" truth["train"]["category_variants"] = ["country", "plan"] truth["train"]["numeric_text"] = ["monthly_spend"] truth["train"]["disguised_missing"] = ["age", "country", "income"] truth["train"]["outliers"] = ["income"] truth["train"]["mixed_date_formats"] = ["signup_date"] truth["train"]["leakage"] = ["cancellation_reason"] truth["train"]["constant"] = ["account_status"] truth["train"]["id_columns"] = ["customer_id"] truth["train"]["pii"] = ["email"] truth["train"]["missing"] = ["referral_code"] # 8. exact duplicate rows (double export) and conflicting duplicates (same customer, different label) dups = [dict(r) for r in rng.sample(rows[:n], 40)] conflicts = [] for r in rng.sample(rows[:n], 12): c = dict(r) c["churned"] = 1 - c["churned"] c["cancellation_reason"] = rng.choice(REASONS) if c["churned"] else "" conflicts.append(c) rows += dups + conflicts truth["train"]["exact_duplicates"] = 40 truth["train"]["conflicting_labels"] = 12 # ---------------- test file (collected later) ---------------- trows = [make_customer(rng, 5000 + i, test=True) for i in range(600)] for r in trows: r["monthly_spend"] = "${:,.2f}".format(r["monthly_spend"]) # 9. 30 training customers leaked into the test file leaked = [dict(r) for r in rng.sample(rows[:n], 30)] trows += leaked truth["test"]["overlap_rows"] = 30 truth["test"]["shifted"] = ["age"] truth["test"]["unseen_categories"] = ["plan"] os.makedirs(OUT, exist_ok=True) for name, data in [("demo_train.csv", rows), ("demo_test.csv", trows)]: with open(os.path.join(OUT, name), "w", newline="") as f: w = csv.writer(f) w.writerow(COLUMNS) for r in data: w.writerow([r[c] for c in COLUMNS]) with open(os.path.join(OUT, "demo_truth.json"), "w") as f: json.dump(truth, f, indent=2) print(f"wrote {len(rows)} training rows and {len(trows)} test rows to {os.path.normpath(OUT)}", file=sys.stderr) if __name__ == "__main__": main()