Birthrate Example

The Birthrate data set contains annual U.S. birth rates from 1917 to 2003.

birthrates <- read.csv("birthrates.csv")
head(birthrates)
##   Year Birthrate
## 1 1917     183.1
## 2 1918     183.9
## 3 1919     163.1
## 4 1920     179.5
## 5 1921     181.4
## 6 1922     173.4
plot(birthrates, pch = 19, col = "blue",
     xlab = "Year", ylab = "Birth rate")

The series exhibits a highly nonlinear trend. We first fit orthogonal polynomial regressions of degrees 3 and 6:

par(mfrow = c(1, 2))

poly3.fit <- lm(Birthrate ~ poly(Year, 3), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, poly3.fit$fitted.values, col = "magenta", lwd = 2)
title("Degree = 3")

poly6.fit <- lm(Birthrate ~ poly(Year, 6), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, poly6.fit$fitted.values, col = "magenta", lwd = 2)
title("Degree = 6")

The degree-6 polynomial improves on the cubic fit but still fails to capture the sharp changes in the series. We next try simple piecewise polynomials (constant and linear) with three interior knots:

myknots <- c(1936, 1960, 1978)
bounds  <- c(1917, myknots, 2003)

# Piecewise constant
mybasis <- cbind(
  x1 = (birthrates$Year < myknots[1]),
  x2 = (birthrates$Year >= myknots[1]) * (birthrates$Year < myknots[2]),
  x3 = (birthrates$Year >= myknots[2]) * (birthrates$Year < myknots[3]),
  x4 = (birthrates$Year >= myknots[3])
)
const.fit <- lm(birthrates$Birthrate ~ . - 1, data = data.frame(mybasis))

plot(birthrates, pch = 19, col = "blue")
abline(v = myknots, lty = 2)
title("Piecewise constant")
for (k in 1:4) {
  segments(bounds[k], const.fit$coefficients[k],
           bounds[k + 1], const.fit$coefficients[k],
           col = "magenta", lwd = 3)
}

# Piecewise linear (discontinuous)
mybasis <- cbind(
  x1  = (birthrates$Year < myknots[1]),
  x2  = (birthrates$Year >= myknots[1]) * (birthrates$Year < myknots[2]),
  x3  = (birthrates$Year >= myknots[2]) * (birthrates$Year < myknots[3]),
  x4  = (birthrates$Year >= myknots[3]),
  x11 = birthrates$Year * (birthrates$Year < myknots[1]),
  x21 = birthrates$Year * (birthrates$Year >= myknots[1]) * (birthrates$Year < myknots[2]),
  x31 = birthrates$Year * (birthrates$Year >= myknots[2]) * (birthrates$Year < myknots[3]),
  x41 = birthrates$Year * (birthrates$Year >= myknots[3])
)
line.fit <- lm(birthrates$Birthrate ~ . - 1, data = data.frame(mybasis))

plot(birthrates, pch = 19, col = "blue")
abline(v = myknots, lty = 2)
title("Piecewise linear")
for (k in 1:4) {
  segments(bounds[k],
           line.fit$coefficients[k] + bounds[k] * line.fit$coefficients[k + 4],
           bounds[k + 1],
           line.fit$coefficients[k] + bounds[k + 1] * line.fit$coefficients[k + 4],
           col = "magenta", lwd = 3)
}

These piecewise fits are local, but the discontinuities at the knots produce an unsatisfactory overall curve.

Regression Splines

A better approach is to use splines, which enforce continuity (and, for cubic splines, continuity of derivatives). We begin with a linear B-spline:

bsplines.lin.fit <- lm(Birthrate ~ bs(Year, degree = 1, knots = myknots), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, bsplines.lin.fit$fitted.values, col = "magenta", lwd = 3)
title("Linear B-spline")

and then a cubic B-spline with the same three knots:

bsplines.3.fit <- lm(Birthrate ~ bs(Year, degree = 3, knots = myknots), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, bsplines.3.fit$fitted.values, col = "magenta", lwd = 3)
title("Cubic B-spline (3 knots)")

Instead of specifying the knots directly we can request a target number of degrees of freedom; bs() then places the knots at appropriate quantiles:

bsplines.df <- lm(Birthrate ~ bs(Year, df = 6), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, bsplines.df$fitted.values, col = "magenta", lwd = 3)
title("Cubic B-spline (df = 6)")

Natural cubic splines impose the additional constraint that the function is linear beyond the boundary knots:

ns.fit <- lm(Birthrate ~ ns(Year, df = 6), data = birthrates)
plot(birthrates, pch = 19, col = "blue")
lines(birthrates$Year, ns.fit$fitted.values, col = "magenta", lwd = 3)
title("Natural cubic spline (df = 6)")

A Simulated Example of Smoothing Splines

We now reproduce the classic illustration from Section 5.5.2 of Elements of Statistical Learning. The true regression function is \[ f(x)=\frac{\sin\bigl(12(x+0.2)\bigr)}{x+0.2},\qquad x\in[0,1]. \] We generate noisy observations from this curve:

set.seed(598)
n <- 100
x <- sort(runif(n))
y <- sin(12 * (x + 0.2)) / (x + 0.2) + rnorm(n, 0, 1)

plot(x, y, col = "blue", pch = 16,
     xlab = "x", ylab = "y")
xx <- seq(0, 1, length = 200)
yy <- sin(12 * (xx + 0.2)) / (xx + 0.2)
lines(xx, yy, col = "gray50", lwd = 2)

A smoothing spline is obtained with the function smooth.spline:

fit5 <- smooth.spline(x, y, df = 5)
plot(x, y, col = "blue", pch = 16,
     xlab = "x", ylab = "y")
lines(xx, yy, col = "gray50", lwd = 2)               # true function
lines(predict(fit5, xx), col = "magenta", lwd = 2, lty = 2)
title("Smoothing spline, df = 5")

Increasing the degrees of freedom produces a more flexible fit:

fit8 <- smooth.spline(x, y, df = 8)
plot(x, y, col = "blue", pch = 16,
     xlab = "x", ylab = "y")
lines(xx, yy, col = "gray50", lwd = 2)
lines(predict(fit8, xx), col = "magenta", lwd = 2, lty = 2)
title("Smoothing spline, df = 8")

Choice of the Smoothing Parameter

In practice \(\lambda\) (or equivalently the degrees of freedom) is chosen by cross-validation. The function smooth.spline reports both the effective degrees of freedom and the leverages (diagonal entries of the smoother matrix):

fit9 <- smooth.spline(x, y, df = 9)
fit9$df
## [1] 9.001417
sum(fit9$lev)          # should equal df (up to rounding)
## [1] 9.001417

By default the function returns the generalized cross-validation (GCV) score. Setting cv = TRUE yields the ordinary leave-one-out CV score instead:

fit9$cv                                # GCV
## [1] FALSE
fit.cv <- smooth.spline(x, y, df = 9, cv = TRUE)
fit.cv$cv                              # LOOCV
## [1] TRUE

Both criteria can be examined over a grid of degrees of freedom to select a final model.