Section 5.3 Python Example¶
To illustrate the ideas in this chapter, we will predict house prices from a set of features including size, beds, zip, and a wealth index. This is a simulated example for illustration purposes, and the data can be found and downloaded in Section 5.3.
The response $y$ (house price) is obviously continuous, so we will fit linear and nonlinear regression models, this is the specified $\phi$ from the previous chapters, and then we will employ a neural net to learn $\phi_c$ using a squared error loss. In the end we will compare the results on the testing set.
Specifically, we will fit:
OLS on the raw features, $\phi(x)=x$;
OLS on a hand-built $\phi$ (a corner in size, and interaction $\mathrm{beds}\times\mathrm{wealth}$);
OLS on a cubic in
sizeplus the other raw features;a one-hidden-layer net with a linear last layer, two widths, with weight decay.
We will report $\frac1n\sum_i(\hat y_i-y_i)^2$ on a test set.
The approach used here is not a clone of nnet. You will also see that there are even differences in the results. It is not only the algorithms that differ, but also the training and testing data (even though the seed to split is the same). This is ok since the goal is not exactly to mimic what R is doing, but to perform the same type of analysis "in spirit" with the tools available in each language.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import mean_squared_error
plt.rcParams.update({
"figure.figsize": (7, 4.2),
"axes.grid": False,
"font.size": 11,
})
rng = np.random.default_rng(598)
def relu(z):
return np.maximum(z, 0)
def mse(y, yhat):
return mean_squared_error(y, yhat)
houses = pd.read_csv("houses.csv")
houses["zip"] = houses["zip"].astype("category")
houses.head()
| size | beds | zip | wealth | price | |
|---|---|---|---|---|---|
| 0 | 3426 | 6 | 94301 | 1.528712 | 1290.654787 |
| 1 | 3537 | 5 | 94022 | 1.296086 | 1205.212185 |
| 2 | 670 | 1 | 94022 | 0.556931 | 197.812662 |
| 3 | 1603 | 3 | 94301 | 0.734663 | 467.512398 |
| 4 | 2604 | 4 | 94022 | 1.152469 | 896.170746 |
zip is a category so it enters as dummy variables, matching factor(zip) in R.
Exploratory plots¶
fig, ax = plt.subplots()
sc = ax.scatter(
houses["size"], houses["price"],
c=houses["beds"], cmap="viridis", alpha=0.55, s=18,
)
# loess-like visual: rolling mean on size-sorted points
ord_ = np.argsort(houses["size"].to_numpy())
s_sorted = houses["size"].to_numpy()[ord_]
p_sorted = houses["price"].to_numpy()[ord_]
win = 40
kern = np.ones(win) / win
p_smooth = np.convolve(p_sorted, kern, mode="same")
ax.plot(s_sorted, p_smooth, color="black", lw=1.2)
ax.axvline(1100, ls="--", color="grey")
fig.colorbar(sc, ax=ax, label="beds")
ax.set_xlabel("size (ft²)")
ax.set_ylabel("price ($1000s)")
ax.set_title("Price vs size: a floor, then a slope")
fig.suptitle("Dashed line: 1100 ft², where the size effect starts in the hidden law",
fontsize=9, y=0.02)
plt.tight_layout()
plt.show()
We can see that the straight line does not have a very good fit. There is a corner/elbow on the lower left of the plot.
Remark. R used geom_smooth(..., method = "loess"). Here this is replaced by a moving average stands in. The dashed line at 1100 is the same.
zips = list(houses["zip"].cat.categories)
fig, axes = plt.subplots(2, 2, figsize=(8, 6), sharex=True, sharey=True)
for ax, z in zip(axes.ravel(), zips):
sub = houses[houses["zip"] == z]
ax.scatter(sub["wealth"], sub["price"], alpha=0.4, s=16)
if len(sub) > 2:
lr = LinearRegression().fit(sub[["wealth"]], sub["price"])
xs = np.linspace(sub["wealth"].min(), sub["wealth"].max(), 50)
xs_df = pd.DataFrame(xs, columns=["wealth"])
ax.plot(xs, lr.predict(xs_df), color="steelblue")
ax.set_title(f"zip {z}")
fig.supxlabel("neighbourhood wealth (index)")
fig.supylabel("price ($1000s)")
fig.suptitle("Wealth vs price, by zip")
plt.tight_layout()
plt.show()
Training vs testing¶
We start by scaling the numeric columns in the training set. For neural net, we also need to scale $y$.
n = len(houses)
n_train = int(np.floor(0.75 * n))
index = rng.choice(n, size=n_train, replace=False)
train = houses.iloc[index].copy()
test = houses.drop(index=houses.index[index]).copy()
def design_matrix(df, dummy_cols=None):
"""size + beds + zip dummies (drop first) + wealth. Matches model.matrix(...,)[, -1]."""
X_num = df[["size", "beds", "wealth"]].to_numpy(dtype=float)
dummies = pd.get_dummies(df["zip"], drop_first=True, dtype=float)
if dummy_cols is None:
dummy_cols = list(dummies.columns)
else:
dummies = dummies.reindex(columns=dummy_cols, fill_value=0.0)
X = np.column_stack([X_num, dummies.to_numpy()])
names = ["size", "beds", "wealth"] + [f"zip_{c}" for c in dummy_cols]
return X, names, dummy_cols
X_tr_raw, xnames, dummy_cols = design_matrix(train)
X_te_raw, _, _ = design_matrix(test, dummy_cols=dummy_cols)
x_scaler = StandardScaler().fit(X_tr_raw)
y_scaler = StandardScaler().fit(train[["price"]])
X_tr = x_scaler.transform(X_tr_raw)
X_te = x_scaler.transform(X_te_raw)
y_tr = train["price"].to_numpy()
y_te = test["price"].to_numpy()
y_tr_sc = y_scaler.transform(train[["price"]]).ravel()
def unscale_y(y_sc):
return y_scaler.inverse_transform(np.asarray(y_sc).reshape(-1, 1)).ravel()
print(train.shape, test.shape)
print("columns:", xnames)
(600, 5) (200, 5) columns: ['size', 'beds', 'wealth', 'zip_94301', 'zip_94305', 'zip_95112']
Remark. The 75/25 split uses seed 598 but R’s sample() and NumPy’s RNG are different algorithms. Row membership will not match the R results. The comparison will.
We are now ready to run the different types of "regressions". For each model, we keep the MSE on the testing set to compare in the end.
1. Raw features¶
$f_\beta(x)=\beta^\top x$ on size, beds, zip dummies, and wealth.
fit_raw = LinearRegression().fit(X_tr_raw, y_tr)
pred_raw_tr = fit_raw.predict(X_tr_raw)
pred_raw_te = fit_raw.predict(X_te_raw)
mse_raw = {"train": mse(y_tr, pred_raw_tr), "test": mse(y_te, pred_raw_te)}
coef_raw = pd.Series(
np.r_[fit_raw.intercept_, fit_raw.coef_],
index=["(Intercept)"] + xnames,
)
print(coef_raw)
mse_raw
(Intercept) -272.244225 size 0.263676 beds 51.923385 wealth 204.391743 zip_94301 -0.062977 zip_94305 50.472657 zip_95112 7.738739 dtype: float64
{'train': 2229.9684938364885, 'test': 2586.849943761082}
2. Hand-built $\phi$¶
In the simulated data, we have a a floor on size at $1100$ ft$^2$, and $\mathrm{beds}\times\mathrm{wealth}$. So, we fit an OLS that matches this specific behavior:
def phi_matrix(df, dummy_cols):
size_corner = relu(df["size"].to_numpy() - 1100)
beds = df["beds"].to_numpy(dtype=float)
wealth = df["wealth"].to_numpy(dtype=float)
beds_wealth = beds * wealth
dummies = pd.get_dummies(df["zip"], drop_first=True, dtype=float)
dummies = dummies.reindex(columns=dummy_cols, fill_value=0.0)
X = np.column_stack([size_corner, beds, wealth, beds_wealth, dummies.to_numpy()])
names = ["size_corner", "beds", "wealth", "beds_wealth"] + [f"zip_{c}" for c in dummy_cols]
return X, names
X_tr_phi, phi_names = phi_matrix(train, dummy_cols)
X_te_phi, _ = phi_matrix(test, dummy_cols)
fit_phi = LinearRegression().fit(X_tr_phi, y_tr)
pred_phi_tr = fit_phi.predict(X_tr_phi)
pred_phi_te = fit_phi.predict(X_te_phi)
mse_phi = {"train": mse(y_tr, pred_phi_tr), "test": mse(y_te, pred_phi_te)}
coef_phi = pd.Series(
np.r_[fit_phi.intercept_, fit_phi.coef_],
index=["(Intercept)"] + phi_names,
)
print(coef_phi)
mse_phi
(Intercept) 61.367506 size_corner 0.281319 beds 27.017696 wealth 120.194488 beds_wealth 26.172821 zip_94301 -3.736079 zip_94305 51.936791 zip_95112 8.740697 dtype: float64
{'train': 1431.9442745882393, 'test': 1341.3099248634621}
3. Cubic in size; Raw in Other Features¶
poly = PolynomialFeatures(degree=3, include_bias=False)
size_tr_p = poly.fit_transform(train[["size"]])
size_te_p = poly.transform(test[["size"]])
# orthogonal-ish scaling of the three size powers, then append beds, wealth, zip
rest_tr = X_tr_raw[:, 1:] # beds, wealth, zip dummies (size was col 0)
rest_te = X_te_raw[:, 1:]
X_tr_poly = np.column_stack([size_tr_p, rest_tr])
X_te_poly = np.column_stack([size_te_p, rest_te])
fit_poly = LinearRegression().fit(X_tr_poly, y_tr)
pred_poly_tr = fit_poly.predict(X_tr_poly)
pred_poly_te = fit_poly.predict(X_te_poly)
mse_poly = {"train": mse(y_tr, pred_poly_tr), "test": mse(y_te, pred_poly_te)}
mse_poly
{'train': 5600.236639106865, 'test': 6053.23249419714}
Remark. R’s poly(size, 3) uses orthogonal polynomials. PolynomialFeatures uses raw powers $x,x^2,x^3$. The fitted span of polynomials of degree $\le 3$ is the same, so predictions match up to numerics, but coefficients will not.
4. One-Hidden-Layer Net¶
sklearn.neural_network.MLPRegressor with activation="logistic" and solver="lbfgs" is the closest stand-in for R nnet(..., linout = TRUE):
$$ a=\sigma(W^{[1]}x+b^{[1]}),\qquad \hat y = W^{[2]}a+b^{[2]}, $$
sigmoid hidden units, linear last layer, L2 penalty (alpha $\leftrightarrow$ decay).
The net sees only the scaled raw design, not size_corner or beds_wealth.
def fit_one_mlp(hidden=8, alpha=0.001, random_state=0, max_iter=2000):
est = MLPRegressor(
hidden_layer_sizes=(hidden,),
activation="logistic",
solver="lbfgs",
alpha=alpha,
max_iter=max_iter,
random_state=random_state,
)
est.fit(X_tr, y_tr_sc)
return est
def pick_best(hidden, alpha, n_starts, seed0):
models = [fit_one_mlp(hidden, alpha, random_state=seed0 + i) for i in range(n_starts)]
losses = [m.loss_ for m in models]
return models[int(np.argmin(losses))]
nn8 = pick_best(hidden=8, alpha=0.001, n_starts=10, seed0=598)
pred_nn8_tr = unscale_y(nn8.predict(X_tr))
pred_nn8_te = unscale_y(nn8.predict(X_te))
mse_nn8 = {"train": mse(y_tr, pred_nn8_tr), "test": mse(y_te, pred_nn8_te)}
nn16 = pick_best(hidden=16, alpha=0.002, n_starts=8, seed0=5981)
pred_nn16_tr = unscale_y(nn16.predict(X_tr))
pred_nn16_te = unscale_y(nn16.predict(X_te))
mse_nn16 = {"train": mse(y_tr, pred_nn16_tr), "test": mse(y_te, pred_nn16_te)}
nn_wide = pick_best(hidden=24, alpha=1e-5, n_starts=6, seed0=5982)
pred_wide_tr = unscale_y(nn_wide.predict(X_tr))
pred_wide_te = unscale_y(nn_wide.predict(X_te))
mse_wide = {"train": mse(y_tr, pred_wide_tr), "test": mse(y_te, pred_wide_te)}
{"nn8": mse_nn8, "nn16": mse_nn16, "nn_wide": mse_wide}
{'nn8': {'train': 1344.3471855208793, 'test': 1732.14431478915},
'nn16': {'train': 1271.6224500290948, 'test': 1815.4236209469027},
'nn_wide': {'train': 1365.6640158305581, 'test': 1627.9239107807057}}
Train and test MSE¶
board = pd.DataFrame({
"method": [
"OLS, raw features",
"OLS, hand-built φ (corner + interaction)",
"OLS, cubic in size",
"NN, 8 hidden units",
"NN, 16 hidden units",
"NN, 24 hidden units, tiny decay",
],
"chapter": [
"previous chapter, φ = id",
"previous chapter, φ specified",
"previous chapter, generic basis",
"this chapter",
"this chapter",
"this chapter, extra capacity",
],
"train_MSE": [
mse_raw["train"], mse_phi["train"], mse_poly["train"],
mse_nn8["train"], mse_nn16["train"], mse_wide["train"],
],
"test_MSE": [
mse_raw["test"], mse_phi["test"], mse_poly["test"],
mse_nn8["test"], mse_nn16["test"], mse_wide["test"],
],
})
board["test_RMSE"] = np.sqrt(board["test_MSE"])
board = board.sort_values("test_MSE").reset_index(drop=True)
board
| method | chapter | train_MSE | test_MSE | test_RMSE | |
|---|---|---|---|---|---|
| 0 | OLS, hand-built φ (corner + interaction) | previous chapter, φ specified | 1431.944275 | 1341.309925 | 36.623898 |
| 1 | NN, 24 hidden units, tiny decay | this chapter, extra capacity | 1365.664016 | 1627.923911 | 40.347539 |
| 2 | NN, 8 hidden units | this chapter | 1344.347186 | 1732.144315 | 41.619038 |
| 3 | NN, 16 hidden units | this chapter | 1271.622450 | 1815.423621 | 42.607788 |
| 4 | OLS, raw features | previous chapter, φ = id | 2229.968494 | 2586.849944 | 50.861085 |
| 5 | OLS, cubic in size | previous chapter, generic basis | 5600.236639 | 6053.232494 | 77.802522 |
order = board.sort_values("test_MSE")
colors = {
"previous chapter, φ = id": "#1b9e77",
"previous chapter, φ specified": "#d95f02",
"previous chapter, generic basis": "#7570b3",
"this chapter": "#e7298a",
"this chapter, extra capacity": "#66a61e",
}
fig, ax = plt.subplots(figsize=(7, 4.2))
ax.barh(
order["method"], order["test_MSE"],
color=[colors[c] for c in order["chapter"]],
)
ax.set_xlabel("test MSE (lower is better)")
ax.set_title("Test MSE")
ax.invert_yaxis()
plt.tight_layout()
plt.show()
The results we obtain are not surprising. The "hand-built" $\phi$ is the best, since it captures exactly the corner points. Obviously, this is not easy to guess in practice (when one does not know whether even to expect a corner). An experienced statistician might pick it up by looking at the residual plots.
A wider net with a tiny decay is quite close to the hand-picked OLS, without being told the columns which shows that the NN learned the shape of $phi$ with the two other NN closely follow.
The cubic OLS in size which fakes a bend, but misses the interaction has the worst fit even compared to the OLS over the raw features. There is a big difference from the results in R - the main reason is that the polynomials in R were standardized (avoiding collinearity issues) which is not the case here.
preds = pd.DataFrame({
"y": y_te,
"OLS raw": pred_raw_te,
"OLS φ": pred_phi_te,
"OLS cubic": pred_poly_te,
"NN 8": pred_nn8_te,
"NN 16": pred_nn16_te,
})
fig, axes = plt.subplots(2, 3, figsize=(9, 6))
axes = axes.ravel()
for ax, col in zip(axes, ["OLS raw", "OLS φ", "OLS cubic", "NN 8", "NN 16"]):
ax.scatter(preds["y"], preds[col], alpha=0.4, s=12)
lo = min(preds["y"].min(), preds[col].min())
hi = max(preds["y"].max(), preds[col].max())
ax.plot([lo, hi], [lo, hi], color="grey")
ax.set_title(col)
ax.set_aspect("equal", adjustable="box")
axes[-1].axis("off")
fig.supxlabel("actual price")
fig.supylabel("predicted price")
fig.suptitle("Test set: predicted vs actual")
plt.tight_layout()
plt.show()
What raw OLS missed¶
fig, ax = plt.subplots()
ax.axhline(0, color="grey")
ax.axvline(1100, ls="--", color="grey")
ax.scatter(test["size"], y_te - pred_raw_te, alpha=0.45, s=16)
ord_ = np.argsort(test["size"].to_numpy())
s = test["size"].to_numpy()[ord_]
r = (y_te - pred_raw_te)[ord_]
win = 25
if len(s) > win:
ax.plot(s, np.convolve(r, np.ones(win)/win, mode="same"), color="firebrick")
ax.set_xlabel("size (ft²)")
ax.set_ylabel("y − ŷ_raw")
ax.set_title("Raw OLS residuals vs size: leftover corner")
plt.tight_layout()
plt.show()
fig, axes = plt.subplots(1, 2, figsize=(8, 3.8), sharey=True)
for ax, resid, title in [
(axes[0], y_te - pred_phi_te, "OLS, hand-built φ"),
(axes[1], y_te - pred_nn8_te, "NN, 8 units"),
]:
ax.axhline(0, color="grey")
ax.scatter(test["size"], resid, alpha=0.4, s=14)
ord_ = np.argsort(test["size"].to_numpy())
s = test["size"].to_numpy()[ord_]
r = np.asarray(resid)[ord_]
win = 25
if len(s) > win:
ax.plot(s, np.convolve(r, np.ones(win)/win, mode="same"), color="steelblue")
ax.set_title(title)
ax.set_xlabel("size (ft²)")
axes[0].set_ylabel("y − ŷ")
fig.suptitle("Residuals vs size after a specified φ and after a net")
plt.tight_layout()
plt.show()