Titanic EDA with Numpy, a first pass

import numpy as np

filename = 'titanic.csv'

try:
    data = np.genfromtxt(
        filename,
        delimiter=',',
        dtype=None,
        encoding='utf-8',
        names=True
    )
except ValueError as e:
    print("\n".join(e.args[0].split("\n")[:3]), "\n\t...")

with open(filename, "r") as file:
    for i in range(2):
        print(file.readline())

import csv

with open("titanic.csv", newline="", encoding="utf-8") as source, \
     open("titanic.tsv", "w", encoding="utf-8") as target:
    for row in csv.reader(source):          # reads "Braund, Mr. Owen Harris" as one field
        assert not any("\t" in field for field in row)   # otherwise we would need quotation marks again
        target.write("\t".join(row) + "\n")

filename = 'titanic.tsv'

data = np.genfromtxt(
    filename,
    delimiter='\t',
    dtype=None,
    encoding='utf-8',
    names=True
)

print("Columns:", data.dtype.names)
print("First 5 rows:")
print(data[:5])

survived = data['Survived'].astype(int)
pclass = data['Pclass'].astype(int)
sex = data['Sex']
age = data['Age']  # may contain nan
sibsp = data['SibSp'].astype(int)
parch = data['Parch'].astype(int)
embarked = data['Embarked']

used_columns = {'Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Embarked'}
all_columns = set(data.dtype.names)
unused_columns = all_columns - used_columns
print("Unused columns:", unused_columns)

print("\nNumber of missing age values:", np.sum(np.isnan(age)))

# replace missing age values by the median (just as an example)
non_nan_age = age[~np.isnan(age)]
age_filled = age.copy()
age_filled[np.isnan(age)] = np.median(non_nan_age)

print("Missing age values replaced by the median:", np.median(non_nan_age))

# new feature
def survival_rate(mask):
    total = np.sum(mask)
    survived_total = np.sum(survived[mask])
    rate = survived_total / total if total > 0 else 0
    return total, survived_total, rate

# group by Sex
for s in np.unique(sex):
    mask = (sex == s)
    total, survived_s, rate = survival_rate(mask)
    print(f"Sex: {s:6s} | Total: {total:3d} | Survived: {survived_s:3d} | Rate: {rate:.2f}")

# group by passenger class
for c in np.unique(pclass):
    mask = (pclass == c)
    total, survived_c, rate = survival_rate(mask)
    print(f"Pclass: {c} | Total: {total:3d} | Survived: {survived_c:3d} | Rate: {rate:.2f}")

# Binning
missing = np.isnan(age)  # keep track of the missing entries separately, no sentinel value!

age_group = np.full(age.shape, 'Unknown', dtype='<U10')
age_group[~missing] = 'Adult'
age_group[~missing & (age < 13)] = 'Child'
age_group[~missing & (age >= 60)] = 'Senior'

for grp in np.unique(age_group):
    mask = (age_group == grp)
    total, survived_grp, rate = survival_rate(mask)
    print(f"Age Group: {grp:7s} | Total: {total:3d} | Survived: {survived_grp:3d} | Rate: {rate:.2f}")

import matplotlib.pyplot as plt

labels = []
rates = []
for s in np.unique(sex):
    mask = (sex == s)
    _, _, rate = survival_rate(mask)
    labels.append(s)
    rates.append(rate)

plt.figure(figsize=(5, 3))
plt.bar(labels, rates, color='skyblue')
plt.title('Survival rate by sex')
plt.ylabel('Rate')
plt.ylim(0, 1)
plt.grid(axis='y')
plt.show()

prediction = (sex == 'female')    # prediction: survived
truth = (survived == 1)           # ground truth

tp = np.sum( prediction &  truth)   # true positive:  predicted survived, did survive
fp = np.sum( prediction & ~truth)   # false positive: predicted survived, died
fn = np.sum(~prediction &  truth)   # false negative: predicted died, did survive
tn = np.sum(~prediction & ~truth)   # true negative:  predicted died, did die

print("                   predicted: died  survived")
print(f"actually died:                {tn:4d}      {fp:4d}")
print(f"actually survived:            {fn:4d}      {tp:4d}")
assert tp + fp + fn + tn == len(survived)

n = len(survived)

def summary_measures(prediction):
    tp = np.sum( prediction &  truth); fp = np.sum( prediction & ~truth)
    fn = np.sum(~prediction &  truth); tn = np.sum(~prediction & ~truth)
    accuracy  = (tp + tn) / n
    precision = tp / (tp + fp) if tp + fp > 0 else np.nan   # undefined without a positive prediction
    recall    = tp / (tp + fn)
    return accuracy, precision, recall

for name, pred in [("women survive", sex == 'female'),
                   ("nobody survives", np.zeros(n, dtype=bool))]:
    acc, prec, rec = summary_measures(pred)
    print(f"{name:18s} Accuracy {acc:.3f}   Precision {prec:.3f}   Recall {rec:.3f}")

acc, prec, rec = summary_measures((sex == 'female') | (~missing & (age < 13)))
print(f"Women and children Accuracy {acc:.3f}   Precision {prec:.3f}   Recall {rec:.3f}")

p_survived_if_female = np.mean(survived[sex == 'female'])         # P(survived | female) = precision
p_female_if_survived = np.mean(sex[survived == 1] == 'female')    # P(female | survived) = recall
p_female = np.mean(sex == 'female')    # base rate: proportion of women
p_survived = np.mean(survived)         # base rate: survival rate

print(f"P(survived | female) = {p_survived_if_female:.3f}")
print(f"P(female | survived) = {p_female_if_survived:.3f}")
print(f"Conversion: {p_female_if_survived:.3f} * {p_survived:.3f} / {p_female:.3f}"
      f" = {p_female_if_survived * p_survived / p_female:.3f}")

is_female = (sex == 'female').astype(int)
is_male = (sex == 'male').astype(int)

corf = np.corrcoef(survived, is_female)[0, 1]
corm = np.corrcoef(survived, is_male)[0, 1]

print(f"Correlation between being female and survival: {corf:.2f}")
print(f"Correlation between being male and survival: {corm:.2f}")