{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"# Model Selection for Polynomial Regression"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"attributes": {
"classes": [],
"id": "",
"n": "1"
}
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import d2l\n",
"from mxnet import autograd, gluon, nd\n",
"from mxnet.gluon import data as gdata, loss as gloss, nn\n",
"\n",
"def semilogy(x_vals, y_vals, x_label, y_label, x2_vals=None, y2_vals=None,\n",
" legend=None, figsize=(8, 4)):\n",
" d2l.set_figsize(figsize)\n",
" d2l.plt.xlabel(x_label)\n",
" d2l.plt.ylabel(y_label)\n",
" d2l.plt.semilogy(x_vals, y_vals)\n",
" if x2_vals and y2_vals:\n",
" d2l.plt.semilogy(x2_vals, y2_vals, linestyle=':')\n",
" d2l.plt.legend(legend)"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Generating Data Sets\n",
"\n",
"$$y = 5 + 1.2x - 3.4\\frac{x^2}{2!} + 5.6 \\frac{x^3}{3!} + \\epsilon \\text{ where }\n",
"\\epsilon \\sim \\mathcal{N}(0,0.1)$$"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"attributes": {
"classes": [],
"id": "",
"n": "2"
}
},
"outputs": [],
"source": [
"maxdegree = 20 # maximum degree of the polynomial\n",
"n_train, n_test = 100, 1000 # training and test data set sizes\n",
"true_w = nd.zeros(maxdegree) # allocate lots of empty space\n",
"true_w[0:4] = nd.array([5, 1.2, -3.4, 5.6])\n",
"\n",
"features = nd.random.normal(shape=(n_train + n_test, 1))\n",
"features = nd.random.shuffle(features)\n",
"poly_features = nd.power(features, nd.arange(maxdegree).reshape((1,-1)))\n",
"poly_features = poly_features / (nd.gamma(nd.arange(maxdegree)+1).reshape((1,-1)))\n",
"labels = nd.dot(poly_features, true_w)\n",
"labels += nd.random.normal(scale=0.1, shape=labels.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Defining, Training and Testing Model"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"attributes": {
"classes": [],
"id": "",
"n": "5"
}
},
"outputs": [],
"source": [
"def fit_and_plot(train_features, test_features, train_labels, test_labels):\n",
" net = nn.Sequential()\n",
" # Switch off the bias since we already catered for it in the polynomial features\n",
" net.add(nn.Dense(1, use_bias=False))\n",
" net.initialize()\n",
" batch_size = min(10, train_labels.shape[0])\n",
" train_iter = gdata.DataLoader(gdata.ArrayDataset(\n",
" train_features, train_labels), batch_size, shuffle=True)\n",
" trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})\n",
" num_epochs, loss, train_ls, test_ls = 100, gloss.L2Loss(), [], []\n",
" for _ in range(num_epochs):\n",
" for X, y in train_iter:\n",
" with autograd.record():\n",
" l = loss(net(X), y)\n",
" l.backward()\n",
" trainer.step(batch_size)\n",
" evaluate = lambda X, y: loss(net(X), y).mean().asscalar()\n",
" train_ls.append(evaluate(train_features, train_labels))\n",
" test_ls.append(evaluate(test_features, test_labels))\n",
" semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss',\n",
" range(1, num_epochs + 1), test_ls, ['train', 'test'])"
]
},
{
"cell_type": "markdown",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"### Third-order Polynomial Function Fitting (Normal)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"attributes": {
"classes": [],
"id": "",
"n": "6"
}
},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
"