09-12-hand-written-digits.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "\n",
    "\n",
    "# Recognizing Hand-Written Digits with Neural Networks\n",
    "\n",
    "\n",
    "\n",
    "![image.png](images/author.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "\n",
    "\n",
    "\n",
    "## Recognizing Hand-Written Digits Using Numpy\n",
    "\n",
    "![](images/digits.png)\n",
    "\n",
    "**Each image has 8*8 = 64 pixels**\n",
    "\n",
    "\n",
    "\n",
    "- input = 64\n",
    "    - [0, 0, 1, 0, ..., 0]\n",
    "- batch size = 100\n",
    "- hidden neurons = 50\n",
    "- output = 10\n",
    "- using relu activation function\n",
    "\n",
    "This is a copy of the test set of the UCI ML hand-written digits datasets\n",
    "https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:36:56.165824Z",
     "start_time": "2020-11-24T01:36:56.086829Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "import sys\n",
    "from collections import defaultdict\n",
    "from sklearn import datasets\n",
    "\n",
    "# load data\n",
    "digits = datasets.load_digits()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:36:56.767943Z",
     "start_time": "2020-11-24T01:36:56.763315Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1797 1617\n"
     ]
    }
   ],
   "source": [
    "# prepare training sets\n",
    "N, D_in, H, D_out = 100, 64,  50, 10 # batch size, input, hidden, output dimension\n",
    "k = 0.9 # the fraction traning data\n",
    "learning_rate = 1e-6 # 1e-1\n",
    "L = len(digits.data)\n",
    "l = int(L*k)\n",
    "print(L, l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:36:57.474106Z",
     "start_time": "2020-11-24T01:36:57.442587Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "Batches = {}\n",
    "M = 200 # number of batches\n",
    "for j in range(M):\n",
    "    index=list(np.random.randint(l, size=N)) # randomly sample N data points\n",
    "    y = np.zeros((N, 10))\n",
    "    y[np.arange(N), list(digits.target[index])] = 1\n",
    "    x=digits.data[index]\n",
    "    Batches[j]=[x,y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:36:58.198580Z",
     "start_time": "2020-11-24T01:36:58.193874Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# softmax\n",
    "def softmax(x):\n",
    "    e_x = np.exp(x - np.max(x)) # to avoid inf\n",
    "    return e_x / e_x.sum(axis=0)\n",
    "\n",
    "def softmaxByRow(x):\n",
    "    e_x = np.exp(x - x.max(axis=1, keepdims=True))\n",
    "    return e_x / e_x.sum(axis=1, keepdims=True)\n",
    "\n",
    "# flush print\n",
    "def flushPrint(d):\n",
    "    sys.stdout.write('\\r')\n",
    "    sys.stdout.write(str(d))\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:06.278551Z",
     "start_time": "2020-11-24T01:36:58.890688Z"
    },
    "cell_style": "split",
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch =199"
     ]
    }
   ],
   "source": [
    "w1 = np.random.randn(D_in, H)/H \n",
    "w2 = np.random.randn(H, D_out)/H\n",
    "w1c = w1.copy() # for comprision in viz\n",
    "w2c = w2.copy()\n",
    "Loss=defaultdict(lambda:[])\n",
    "# traning \n",
    "for t in range(200):# epoch_num\n",
    "    flushPrint('epoch ='+str( t))\n",
    "    for j in Batches:\n",
    "        x,y=Batches[j]\n",
    "        # Forward\n",
    "        h = x.dot(w1)\n",
    "        h_relu = np.maximum(h, 0)\n",
    "        y_pred = h_relu.dot(w2)\n",
    "        y_pred_soft=softmaxByRow(y_pred)\n",
    "        # loss\n",
    "        loss = np.square(y_pred_soft-y).sum()\n",
    "        Loss[j].append([t,loss])\n",
    "        # Backprop \n",
    "        grad_y_pred = 2.0 * (y_pred_soft-y)\n",
    "        grad_w2 = h_relu.T.dot(grad_y_pred)\n",
    "        grad_h_relu = grad_y_pred.dot(w2.T)\n",
    "        grad_h = grad_h_relu.copy()\n",
    "        grad_h[h < 0] = 0 \n",
    "        grad_w1 = x.T.dot(grad_h)\n",
    "        # Update weights\n",
    "        w1 -= learning_rate * grad_w1\n",
    "        w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cell_style": "split",
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "![image.png](images/digits2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "https://pytorch.org/tutorials/beginner/pytorch_with_examples.html#pytorch-tensors\n",
    "\n",
    "```\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum().item()\n",
    "    if t % 100 == 99:\n",
    "        print(t, loss)\n",
    "    # Backprop to compute gradients \n",
    "    # of w1 and w2 with respect to loss\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:09.036785Z",
     "start_time": "2020-11-24T01:37:09.028955Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9388888888888889"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test\n",
    "TestData=digits.data[-(L-l):]\n",
    "PredictData=np.maximum(TestData.dot(w1),0).dot(w2)\n",
    "compare=np.argmax(PredictData,axis=1)-digits.target[-(L-l):]\n",
    "Accuracy=list(compare).count(0)/float(len(compare))\n",
    "Accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Recognizing Hand-Written Digits Using Pytorch\n",
    "\n",
    "1. Using relu for only one time\n",
    "2. learning rate = 0.1 \n",
    "3. choose to use MSELoss \n",
    "4. Convert y_batch from the form of [1] to the form of [0,1,0, 0, 0, 0, 0, 0, 0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:13.006325Z",
     "start_time": "2020-11-24T01:37:12.997379Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "from torch import nn, optim, from_numpy\n",
    "import numpy as np\n",
    "\n",
    "Xtrain, Xtest, ytrain, ytest = train_test_split(digits.data, digits.target,\n",
    "                                                train_size = 0.9, test_size = 0.1, random_state=1)\n",
    "\n",
    "Xtrain = torch.tensor(Xtrain, dtype = torch.float32)\n",
    "ytrain = torch.tensor(ytrain, dtype = torch.int64)\n",
    "Xtest = torch.tensor(Xtest, dtype = torch.float32)\n",
    "ytest = torch.tensor(ytest, dtype = torch.int64)\n",
    "\n",
    "\n",
    "batch_size = 100\n",
    "train = torch.utils.data.TensorDataset(Xtrain, ytrain)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "test = torch.utils.data.TensorDataset(Xtest, ytest)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:13.892429Z",
     "start_time": "2020-11-24T01:37:13.885449Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from torch.nn import functional as F\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.l1 = nn.Linear(64, 50)\n",
    "        self.l2 = nn.Linear(50, 10)\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.l1(x))\n",
    "        out = self.l2(out)\n",
    "        y_pred = F.softmax(out, dim = -1)\n",
    "        return y_pred\n",
    "\n",
    "# our model\n",
    "model = Model()\n",
    "criterion = nn.MSELoss(reduction='mean')\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-1) # learning rate is very sensitive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:14.705285Z",
     "start_time": "2020-11-24T01:37:14.701237Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def get_prob(y_batch):\n",
    "    y = np.zeros((len(y_batch), 10))\n",
    "    y[np.arange(len(y_batch)), list(y_batch)] = 1\n",
    "    y = torch.tensor(y, dtype = torch.float32)\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:20.355697Z",
     "start_time": "2020-11-24T01:37:15.677267Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/num_epoch | Loss: 0.1011\n",
      "Epoch: 0/num_epoch | Loss: 0.1043\n",
      "Epoch: 0/num_epoch | Loss: 0.0998\n",
      "Epoch: 0/num_epoch | Loss: 0.0977\n",
      "Epoch: 0/num_epoch | Loss: 0.0963\n",
      "Epoch: 0/num_epoch | Loss: 0.0921\n",
      "Epoch: 0/num_epoch | Loss: 0.0955\n",
      "Epoch: 0/num_epoch | Loss: 0.0981\n",
      "Epoch: 0/num_epoch | Loss: 0.0964\n",
      "Epoch: 0/num_epoch | Loss: 0.0915\n",
      "Epoch: 0/num_epoch | Loss: 0.0840\n",
      "Epoch: 0/num_epoch | Loss: 0.0882\n",
      "Epoch: 0/num_epoch | Loss: 0.0974\n",
      "Epoch: 0/num_epoch | Loss: 0.0893\n",
      "Epoch: 0/num_epoch | Loss: 0.0843\n",
      "Epoch: 0/num_epoch | Loss: 0.0885\n",
      "Epoch: 0/num_epoch | Loss: 0.0624\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "num_epoch = 100\n",
    "for k, epoch in enumerate(range(num_epoch)):\n",
    "    for x_batch, y_batch in train_loader:\n",
    "        # Forward pass: Compute predicted y by passing x to the model\n",
    "        y_pred = model(x_batch)  \n",
    "        y_batch = get_prob(y_batch)\n",
    "        # Compute and print loss\n",
    "        loss = criterion(y_pred, y_batch)\n",
    "        #loss = criterion(torch.max(y_pred, 1)[1], y_batch)\n",
    "        if k % 100 ==0:\n",
    "            print(f'Epoch: {epoch}/num_epoch | Loss: {loss.item():.4f}')\n",
    "        # Zero gradients, perform a backward pass, and update the weights.\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cell_style": "split",
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "\n",
    "\n",
    "<font size = 25>Model Validation</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:22.643892Z",
     "start_time": "2020-11-24T01:37:22.634942Z"
    },
    "cell_style": "split",
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 96 %\n"
     ]
    }
   ],
   "source": [
    "# Test the Model\n",
    "model.eval()  # Change model to 'eval' mode \n",
    "correct = 0\n",
    "total = 0\n",
    "for xval, yval in test_loader:\n",
    "    outputs = model(xval)\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "    total += yval.size(0)\n",
    "    correct += (predicted == yval).sum()\n",
    "\n",
    "print('Test Accuracy: %d %%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Recognizing Hand-Written Digits with CNN Using Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:25.111922Z",
     "start_time": "2020-11-24T01:37:25.075038Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "from torch import nn, optim, from_numpy\n",
    "import numpy as np\n",
    "\n",
    "Xtrain, Xtest, ytrain, ytest = train_test_split(digits.data, digits.target,\n",
    "                                                train_size = 0.9, test_size = 0.1, random_state=1)\n",
    "# reshape the vector of length 64 to a matrix of 8*8\n",
    "Xtrain = [i.reshape(8, 8) for i in Xtrain]\n",
    "Xtest  = [i.reshape(8, 8) for i in Xtest]\n",
    "\n",
    "Xtrain = torch.tensor(Xtrain, dtype = torch.float32)\n",
    "ytrain = torch.tensor(ytrain, dtype = torch.int64)\n",
    "Xtest = torch.tensor(Xtest, dtype = torch.float32)\n",
    "ytest = torch.tensor(ytest, dtype = torch.int64)\n",
    "\n",
    "batch_size = 100\n",
    "train = torch.utils.data.TensorDataset(Xtrain, ytrain)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n",
    "test = torch.utils.data.TensorDataset(Xtest, ytest)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:26.150268Z",
     "start_time": "2020-11-24T01:37:26.146508Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1617, 8, 8])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:27.097234Z",
     "start_time": "2020-11-24T01:37:27.093035Z"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([180, 8, 8])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtest.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:28.024818Z",
     "start_time": "2020-11-24T01:37:28.018840Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.,  0.,  2., 13., 16., 16.,  7.,  0.],\n",
       "        [ 0.,  0., 12., 15., 12., 16., 10.,  0.],\n",
       "        [ 0.,  0., 16.,  9.,  0., 14.,  6.,  0.],\n",
       "        [ 0.,  0.,  3.,  0.,  4., 16.,  1.,  0.],\n",
       "        [ 0.,  0.,  0., 10., 14., 16.,  6.,  0.],\n",
       "        [ 0.,  0.,  3., 16., 16., 11.,  2.,  0.],\n",
       "        [ 0.,  0.,  0.,  9., 14.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  2., 15.,  6.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtrain[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:28.943791Z",
     "start_time": "2020-11-24T01:37:28.938838Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.,  0.,  6., 16., 11.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  9., 16., 16.,  5.,  0.,  0.],\n",
       "        [ 0.,  0.,  8., 16., 16.,  4.,  0.,  0.],\n",
       "        [ 0.,  0., 10., 16., 13.,  0.,  0.,  0.],\n",
       "        [ 0.,  0., 13., 16., 12.,  0.,  0.,  0.],\n",
       "        [ 0.,  0., 10., 16.,  9.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  9., 16., 10.,  0.,  0.,  0.],\n",
       "        [ 0.,  0.,  4., 15., 16.,  3.,  0.,  0.]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtest[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:37:30.124046Z",
     "start_time": "2020-11-24T01:37:30.114647Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# CNN Model (2 conv layer)\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "        self.layer1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 32, kernel_size=3, padding=2), # in_channels = 1, out_channels = 32, kernel_size= 3\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2))\n",
    "        self.layer2 = nn.Sequential(\n",
    "            nn.Conv2d(32, 64, kernel_size=3, padding=2),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2))\n",
    "        self.fc = nn.Linear(3*3*64, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.layer2(out)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out\n",
    "\n",
    "cnn = CNN()\n",
    "# Loss and Optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(cnn.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:38:20.249792Z",
     "start_time": "2020-11-24T01:37:31.666504Z"
    },
    "code_folding": [],
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], Iter [10/16] Loss: 2.5073\n",
      "Epoch [2/100], Iter [10/16] Loss: 2.3236\n",
      "Epoch [3/100], Iter [10/16] Loss: 2.2836\n",
      "Epoch [4/100], Iter [10/16] Loss: 1.9957\n",
      "Epoch [5/100], Iter [10/16] Loss: 1.4108\n",
      "Epoch [6/100], Iter [10/16] Loss: 0.5429\n",
      "Epoch [7/100], Iter [10/16] Loss: 0.3105\n",
      "Epoch [8/100], Iter [10/16] Loss: 0.1456\n",
      "Epoch [9/100], Iter [10/16] Loss: 0.2635\n",
      "Epoch [10/100], Iter [10/16] Loss: 0.0572\n",
      "Epoch [11/100], Iter [10/16] Loss: 0.0202\n",
      "Epoch [12/100], Iter [10/16] Loss: 0.0370\n",
      "Epoch [13/100], Iter [10/16] Loss: 0.0279\n",
      "Epoch [14/100], Iter [10/16] Loss: 0.0654\n",
      "Epoch [15/100], Iter [10/16] Loss: 0.0075\n",
      "Epoch [16/100], Iter [10/16] Loss: 0.0093\n",
      "Epoch [17/100], Iter [10/16] Loss: 0.0052\n",
      "Epoch [18/100], Iter [10/16] Loss: 0.0049\n",
      "Epoch [19/100], Iter [10/16] Loss: 0.0021\n",
      "Epoch [20/100], Iter [10/16] Loss: 0.0009\n",
      "Epoch [21/100], Iter [10/16] Loss: 0.0025\n",
      "Epoch [22/100], Iter [10/16] Loss: 0.0005\n",
      "Epoch [23/100], Iter [10/16] Loss: 0.0009\n",
      "Epoch [24/100], Iter [10/16] Loss: 0.0011\n",
      "Epoch [25/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [26/100], Iter [10/16] Loss: 0.0004\n",
      "Epoch [27/100], Iter [10/16] Loss: 0.0003\n",
      "Epoch [28/100], Iter [10/16] Loss: 0.0003\n",
      "Epoch [29/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [30/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [31/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [32/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [33/100], Iter [10/16] Loss: 0.0003\n",
      "Epoch [34/100], Iter [10/16] Loss: 0.0003\n",
      "Epoch [35/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [36/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [37/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [38/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [39/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [40/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [41/100], Iter [10/16] Loss: 0.0003\n",
      "Epoch [42/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [43/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [44/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [45/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [46/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [47/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [48/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [49/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [50/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [51/100], Iter [10/16] Loss: 0.0002\n",
      "Epoch [52/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [53/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [54/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [55/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [56/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [57/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [58/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [59/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [60/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [61/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [62/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [63/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [64/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [65/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [66/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [67/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [68/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [69/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [70/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [71/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [72/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [73/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [74/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [75/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [76/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [77/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [78/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [79/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [80/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [81/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [82/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [83/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [84/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [85/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [86/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [87/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [88/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [89/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [90/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [91/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [92/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [93/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [94/100], Iter [10/16] Loss: 0.0001\n",
      "Epoch [95/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [96/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [97/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [98/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [99/100], Iter [10/16] Loss: 0.0000\n",
      "Epoch [100/100], Iter [10/16] Loss: 0.0000\n"
     ]
    }
   ],
   "source": [
    "# Train the Model\n",
    "from torch.autograd import Variable\n",
    "\n",
    "num_epoch = 100\n",
    "for epoch in range(num_epoch):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        # reshape the shape of data\n",
    "        images = images.view(len(images), 1, 8, 8)\n",
    "        # Forward + Backward + Optimize\n",
    "        optimizer.zero_grad()\n",
    "        outputs = cnn(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if (i+1) % 10 == 0:\n",
    "            print ('Epoch [%d/%d], Iter [%d/%d] Loss: %.4f'\n",
    "                   %(epoch+1, num_epoch, i+1, len(train)//batch_size, loss.data.item()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-24T01:38:23.252456Z",
     "start_time": "2020-11-24T01:38:23.221875Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 98 %\n"
     ]
    }
   ],
   "source": [
    "# Test the Model\n",
    "model.eval()  # Change model to 'eval' mode \n",
    "correct = 0\n",
    "total = 0\n",
    "for xval, yval in test_loader:\n",
    "    # reshape the shape of data\n",
    "    xval = xval.view(len(xval), 1, 8, 8)\n",
    "    outputs = cnn(xval)\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "    total += yval.size(0)\n",
    "    correct += (predicted == yval).sum()\n",
    "\n",
    "print('Test Accuracy: %d %%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "![image.png](images/end.png)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}