Section 4.5 Python Example¶
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm
from patsy import dmatrix
from pygam import LinearGAM, s
pd.set_option('display.precision', 3)
plt.rcParams['figure.figsize'] = (6, 4)
Birthrate Example¶
The Birthrate data set contains annual U.S. birth rates from 1917 to 2003.
birthrates = pd.read_csv("birthrates.csv")
print(birthrates.head())
Year Birthrate 0 1917 183.1 1 1918 183.9 2 1919 163.1 3 1920 179.5 4 1921 181.4
plt.scatter(birthrates['Year'], birthrates['Birthrate'], color="blue", s=20)
plt.xlabel('Year')
plt.ylabel('Birth rate')
plt.title('U.S. Birthrate (1917–2003)')
plt.show()
The series exhibits a highly nonlinear trend. We first fit orthogonal polynomial regressions of degrees 3 and 6.
X = birthrates["Year"]
y = birthrates["Birthrate"]
def fit_orthopoly(x, y, degree):
# Scale to [-1, 1] and use Legendre polynomials (orthogonal on that interval)
X_poly = np.polynomial.legendre.legvander(
np.interp(x, (x.min(), x.max()), (-1, 1)), degree
)
model = sm.OLS(y, X_poly).fit()
x_grid = np.linspace(x.min(), x.max(), 300)
x_scaled = np.interp(x_grid, (x.min(), x.max()), (-1, 1))
X_grid_poly = np.polynomial.legendre.legvander(x_scaled, degree)
y_pred = model.predict(X_grid_poly)
return x_grid, y_pred
x3, y3 = fit_orthopoly(X, y, 3)
x6, y6 = fit_orthopoly(X, y, 6)
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
axes[0].scatter(X, y, color="blue", s=20)
axes[0].plot(x3, y3, color="magenta", linewidth=2)
axes[0].set_title("Degree = 3")
axes[1].scatter(X, y, color="blue", s=20)
axes[1].plot(x6, y6, color="magenta", linewidth=2)
axes[1].set_title("Degree = 6")
plt.tight_layout()
plt.show()
The degree-6 polynomial improves on the cubic fit but still fails to capture the sharp changes in the series. We next try simple piecewise polynomials (constant and linear) with three interior knots.
years = birthrates["Year"].values
rates = birthrates["Birthrate"].values
myknots = [1936, 1960, 1978]
bounds = [1917] + myknots + [2003]
# ---------- Piecewise constant ----------
X_const = np.column_stack([
(years < myknots[0]),
(years >= myknots[0]) & (years < myknots[1]),
(years >= myknots[1]) & (years < myknots[2]),
(years >= myknots[2])
]).astype(float)
const_model = sm.OLS(rates, X_const).fit()
plt.figure()
plt.scatter(years, rates, color="blue", s=20)
for knot in myknots:
plt.axvline(knot, linestyle="--", color="black")
for k in range(4):
plt.plot(
[bounds[k], bounds[k+1]],
[const_model.params[k]] * 2,
color="magenta", linewidth=3
)
plt.title("Piecewise constant")
plt.show()
# ---------- Piecewise linear (discontinuous) ----------
X_linear = np.column_stack([
(years < myknots[0]),
(years >= myknots[0]) & (years < myknots[1]),
(years >= myknots[1]) & (years < myknots[2]),
(years >= myknots[2]),
years * (years < myknots[0]),
years * ((years >= myknots[0]) & (years < myknots[1])),
years * ((years >= myknots[1]) & (years < myknots[2])),
years * (years >= myknots[2])
]).astype(float)
line_model = sm.OLS(rates, X_linear).fit()
plt.figure()
plt.scatter(years, rates, color="blue", s=20)
for knot in myknots:
plt.axvline(knot, linestyle="--", color="black")
for k in range(4):
plt.plot(
[bounds[k], bounds[k+1]],
line_model.params[k] + np.array([bounds[k], bounds[k+1]]) * line_model.params[k+4],
color="magenta", linewidth=3
)
plt.title("Piecewise linear")
plt.show()
These piecewise fits are local, but the discontinuities at the knots produce an unsatisfactory overall curve.
Regression Splines¶
A better approach is to use splines, which enforce continuity (and, for cubic splines, continuity of derivatives). We begin with a linear B-spline.
X_bs = dmatrix("bs(year, knots=(1936,1960,1978), degree=1, include_intercept=True)",
{"year": birthrates["Year"]}, return_type='dataframe')
model_bs = sm.OLS(birthrates["Birthrate"], X_bs).fit()
plt.figure()
plt.scatter(birthrates["Year"], birthrates["Birthrate"], color="blue", s=20)
plt.plot(birthrates["Year"], model_bs.fittedvalues, color="magenta", linewidth=3)
plt.title("Linear B-spline")
plt.show()
Next we fit a cubic B-spline with the same three knots.
X_bs3 = dmatrix("bs(year, knots=(1936,1960,1978), degree=3, include_intercept=True)",
{"year": birthrates["Year"]}, return_type='dataframe')
model_bs3 = sm.OLS(birthrates["Birthrate"], X_bs3).fit()
plt.figure()
plt.scatter(birthrates["Year"], birthrates["Birthrate"], color="blue", s=20)
plt.plot(birthrates["Year"], model_bs3.fittedvalues, color="magenta", linewidth=3)
plt.title("Cubic B-spline (3 knots)")
plt.show()
Instead of specifying the knots directly we can request a target number of degrees of freedom; the basis constructor then places the knots at appropriate quantiles.
X_bs_df = dmatrix("bs(year, df=6, degree=3, include_intercept=True)",
{"year": birthrates["Year"]}, return_type='dataframe')
model_bs_df = sm.OLS(birthrates["Birthrate"], X_bs_df).fit()
plt.figure()
plt.scatter(birthrates["Year"], birthrates["Birthrate"], color="blue", s=20)
plt.plot(birthrates["Year"], model_bs_df.fittedvalues, color="magenta", linewidth=3)
plt.title("Cubic B-spline (df = 6)")
plt.show()
Natural cubic splines impose the additional constraint that the function is linear beyond the boundary knots.
X_ns = dmatrix("cr(year, df=6, lower_bound=1917, upper_bound=2003)",
{"year": birthrates["Year"]}, return_type='dataframe')
model_ns = sm.OLS(birthrates["Birthrate"], X_ns).fit()
order = np.argsort(birthrates["Year"].values)
plt.figure()
plt.scatter(birthrates["Year"], birthrates["Birthrate"], color="blue", s=20)
plt.plot(birthrates["Year"].values[order],
model_ns.fittedvalues.values[order],
color="magenta", linewidth=3)
plt.title("Natural cubic spline (df = 6)")
plt.show()
Note. The natural-cubic-spline basis implemented by patsy is not identical to the one produced by R’s ns() function, so the fitted curves may differ slightly even when the nominal degrees of freedom are the same.
A Simulated Example of Smoothing Splines¶
We now reproduce the classic illustration from Section 5.5.2 of Elements of Statistical Learning. The true regression function is
$$ f(x) = \frac{\sin\bigl(12(x+0.2)\bigr)}{x+0.2}, \qquad x \in [0,1]. $$
We generate noisy observations from this curve:
np.random.seed(598)
n = 100
x = np.sort(np.random.uniform(0, 1, n))
y = np.sin(12 * (x + 0.2)) / (x + 0.2) + np.random.normal(0, 1, n)
xx = np.linspace(0, 1, 200)
yy = np.sin(12 * (xx + 0.2)) / (xx + 0.2)
plt.scatter(x, y, color="blue", s=25)
plt.plot(xx, yy, color="gray", linewidth=2, label="True function")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()
plt.show()
Fitting a Smoothing Spline¶
In Python we use the LinearGAM class from the pygam package (with a spline term). The argument n_splines plays a role analogous to the degrees of freedom in R’s smooth.spline.
gam5 = LinearGAM(s(0, n_splines=5)).fit(x, y)
print(gam5.summary())
LinearGAM
=============================================== ==========================================================
Distribution: NormalDist Effective DoF: 2.7453
Link Function: IdentityLink Log Likelihood: -185.5807
Number of Samples: 100 AIC: 378.652
AICc: 379.0252
GCV: 2.5902
Scale: 1.5692
Pseudo R-Squared: 0.0429
==========================================================================================================
Feature Function Lambda Rank EDoF P > x Sig. Code
================================= ==================== ============ ============ ============ ============
s(0) [0.6] 5 2.7 9.72e-08 ***
intercept 1 0.0 5.27e-01
==========================================================================================================
Significance codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
WARNING: Fitting splines and a linear function to a feature introduces a model identifiability problem
which can cause p-values to appear significant when they are not.
WARNING: p-values calculated in this manner behave correctly for un-penalized models or models with
known smoothing parameters, but when smoothing parameters have been estimated, the p-values
are typically lower than they should be, meaning that the tests reject the null too readily.
None
/var/folders/dc/kyqykw196wx1lxj4xct07yrr0000gn/T/ipykernel_81317/1714811789.py:2: UserWarning: KNOWN BUG: p-values computed in this summary are likely much smaller than they should be. Please do not make inferences based on these values! Collaborate on a solution, and stay up to date at: github.com/dswah/pyGAM/issues/163 print(gam5.summary())
We plot the fitted curve together with the true function:
fitted5 = gam5.predict(xx)
plt.scatter(x, y, color="blue", s=25, label="Data")
plt.plot(xx, yy, color="gray", linewidth=2, label="True function")
plt.plot(xx, fitted5, color="magenta", linestyle="--", linewidth=2,
label="Smoothing spline (n_splines=5)")
plt.xlabel("x")
plt.ylabel("y")
plt.title("n_splines = 5")
plt.legend()
plt.show()
Increasing the number of splines produces a more flexible fit:
gam8 = LinearGAM(s(0, n_splines=8)).fit(x, y)
fitted8 = gam8.predict(xx)
plt.scatter(x, y, color="blue", s=25, label="Data")
plt.plot(xx, yy, color="gray", linewidth=2, label="True function")
plt.plot(xx, fitted8, color="magenta", linestyle="--", linewidth=2,
label="Smoothing spline (n_splines=8)")
plt.xlabel("x")
plt.ylabel("y")
plt.title("n_splines = 8")
plt.legend()
plt.show()
Choice of the Smoothing Parameter¶
In practice the smoothing parameter (or equivalently the effective degrees of freedom) is chosen by cross-validation. The pygam package provides several built-in criteria. Here we examine the effective degrees of freedom and the leverages (diagonal entries of the smoother matrix).
gam9 = LinearGAM(s(0, n_splines=9)).fit(x, y)
print(gam9.summary())
LinearGAM
=============================================== ==========================================================
Distribution: NormalDist Effective DoF: 4.9457
Link Function: IdentityLink Log Likelihood: -159.9178
Number of Samples: 100 AIC: 331.7269
AICc: 332.6145
GCV: 1.6532
Scale: 1.2275
Pseudo R-Squared: 0.4276
==========================================================================================================
Feature Function Lambda Rank EDoF P > x Sig. Code
================================= ==================== ============ ============ ============ ============
s(0) [0.6] 9 4.9 0.00e+00 ***
intercept 1 0.0 8.72e-05 ***
==========================================================================================================
Significance codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
WARNING: Fitting splines and a linear function to a feature introduces a model identifiability problem
which can cause p-values to appear significant when they are not.
WARNING: p-values calculated in this manner behave correctly for un-penalized models or models with
known smoothing parameters, but when smoothing parameters have been estimated, the p-values
are typically lower than they should be, meaning that the tests reject the null too readily.
None
/var/folders/dc/kyqykw196wx1lxj4xct07yrr0000gn/T/ipykernel_81317/3789651403.py:2: UserWarning: KNOWN BUG: p-values computed in this summary are likely much smaller than they should be. Please do not make inferences based on these values! Collaborate on a solution, and stay up to date at: github.com/dswah/pyGAM/issues/163 print(gam9.summary())
The sum of the leverages equals the effective degrees of freedom (up to numerical rounding).
# Construct the smoother matrix and extract leverages
X_basis = gam9._modelmat(x.reshape(-1, 1))
Xmat = X_basis.toarray()
XtX_inv = np.linalg.pinv(Xmat.T @ Xmat)
H = Xmat @ XtX_inv @ Xmat.T
leverages = np.diag(H)
print("First 10 leverages:", np.round(leverages[:10], 4))
print("Sum of leverages:", leverages.sum())
print("Effective df reported by pygam:", gam9.statistics_['edof'])
First 10 leverages: [0.2974 0.2742 0.2533 0.1547 0.1192 0.1161 0.1103 0.1096 0.1029 0.1026] Sum of leverages: 8.999999999999932 Effective df reported by pygam: 4.945695733987347
We can also compute the generalized cross-validation (GCV) and leave-one-out cross-validation (LOOCV) scores manually:
y_hat = gam9.predict(x)
rss = np.sum((y - y_hat)**2)
edof = gam9.statistics_['edof']
gcv = rss / ((1 - edof / n)**2) / n
print("GCV:", gcv)
loocv = np.mean(((y - y_hat) / (1 - leverages))**2)
print("LOOCV:", loocv)
GCV: 1.5851151455014354 LOOCV: 1.919695670296906
Both criteria can be examined over a grid of n_splines (or smoothing-parameter values) to select a final model.