CNN_Gaussian_Pyramid_6layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the GPU!\n"
     ]
    }
   ],
   "source": [
    "# Detect if we have a GPU available\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "if torch.cuda.is_available():\n",
    "    print(\"Using the GPU!\")\n",
    "else:\n",
    "    print(\"WARNING: Could not find GPU! Using CPU only\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The sample size of training set is:  3556\n",
      "The sample size of testing set is:  889\n",
      "(3556, 1)\n",
      "(889, 1)\n"
     ]
    }
   ],
   "source": [
    "x_train_images = np.load(\"data/x_train_images.npy\")\n",
    "x_test_images = np.load(\"data/x_test_images.npy\")\n",
    "  \n",
    "x_train_images_100 = np.load(\"data/x_train_images_100.npy\")\n",
    "x_test_images_100 = np.load(\"data/x_test_images_100.npy\")\n",
    "\n",
    "x_train_images_50 = np.load(\"data/x_train_images_50.npy\")\n",
    "x_test_images_50 = np.load(\"data/x_test_images_50.npy\")\n",
    "\n",
    "y_train = np.load(\"data/y_train.npy\")\n",
    "y_test = np.load(\"data/y_test.npy\")\n",
    "print(\"The sample size of training set is: \", x_train_images.shape[0])\n",
    "print(\"The sample size of testing set is: \", x_test_images.shape[0])\n",
    "\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    0.336333\n",
       "1    0.325928\n",
       "3    0.251969\n",
       "0    0.085771\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bridge numpy to torch\n",
    "x_train_images_torch = torch.as_tensor(x_train_images).float()\n",
    "x_test_images_torch = torch.as_tensor(x_test_images).float()\n",
    "\n",
    "x_train_images_torch_100 = torch.as_tensor(x_train_images_100).float()\n",
    "x_test_images_torch_100 = torch.as_tensor(x_test_images_100).float()\n",
    "\n",
    "x_train_images_torch_50 = torch.as_tensor(x_train_images_50).float()\n",
    "x_test_images_torch_50 = torch.as_tensor(x_test_images_50).float()\n",
    "\n",
    "y_train_torch = torch.as_tensor(y_train[:,0])\n",
    "y_test_torch = torch.as_tensor(y_test[:,0])\n",
    "n_train = x_train_images.shape[0]\n",
    "n_test = x_test_images.shape[0]\n",
    "# inputs: x_train_nhts, x_train_images, x_test_nhts, x_test_images, y_train, and y_test; \n",
    "K = len(np.unique(y_train))\n",
    "# \n",
    "pd.value_counts(y_train[:,0])/y_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Type 2: CNN with images.\n",
    "# To-Do: Use GPU...\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "        # To-Do: need to have more channels for higher accuracy. \n",
    "        self.conv1 = nn.Conv2d(in_channels=4, out_channels=10, kernel_size=4, padding=2)\n",
    "        self.conv2 = nn.Conv2d(in_channels=10, out_channels=8, kernel_size=4)\n",
    "        self.conv3 = nn.Conv2d(in_channels=8, out_channels=4, kernel_size=4)\n",
    "        # Question: Why is this 48*48 correct? bc 97//2 = 48\n",
    "        self.fc11 = nn.Linear(in_features=4 * 45 * 45, out_features=64)\n",
    "        self.fc12 = nn.Linear(in_features=64, out_features=128)\n",
    "        \n",
    "        self.fc21 = nn.Linear(in_features=4 * 20 * 20, out_features=64)\n",
    "        self.fc22 = nn.Linear(in_features=64, out_features=128)\n",
    "        \n",
    "        self.fc31 = nn.Linear(in_features=4 * 8 * 8, out_features=64)\n",
    "        self.fc32 = nn.Linear(in_features=64, out_features=128)\n",
    "        \n",
    "        self.fcFinal = nn.Linear(in_features=128*3, out_features=K)\n",
    "        self.relu = F.relu\n",
    "        self.pool = F.max_pool2d\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "    def forward(self, x1, x2, x3):\n",
    "        out = self.relu(self.conv1(x1))\n",
    "        out = self.pool(out, 2)\n",
    "        out = self.relu(self.conv2(out))\n",
    "        out = self.pool(out, 2)\n",
    "        out = self.relu(self.conv3(out))\n",
    "        #print(out.shape,\"out1\")\n",
    "        out = out.reshape(out.size(0), -1)\n",
    "        out = self.relu(self.fc11(out))\n",
    "        out = self.fc12(out)\n",
    "        \n",
    "        out2 = self.relu(self.conv1(x2))\n",
    "        out2 = self.pool(out2, 2)\n",
    "        out2 = self.relu(self.conv2(out2))\n",
    "        out2 = self.pool(out2, 2)\n",
    "        out2 = self.relu(self.conv3(out2))\n",
    "        #print(out2.shape,\"out2\")\n",
    "        out2 = out2.reshape(out2.size(0), -1)\n",
    "        out2 = self.relu(self.fc21(out2))\n",
    "        out2 = self.fc22(out2)\n",
    "        \n",
    "        \n",
    "        out3 = self.relu(self.conv1(x3))\n",
    "        out3 = self.pool(out3, 2)\n",
    "        out3 = self.relu(self.conv2(out3))\n",
    "        out3 = self.pool(out3, 2)\n",
    "        out3 = self.relu(self.conv3(out3))\n",
    "        #print(out3.shape,\"out3\")\n",
    "        out3 = out3.reshape(out3.size(0), -1)\n",
    "        out3 = self.relu(self.fc31(out3))\n",
    "        out3 = self.fc32(out3)\n",
    "        \n",
    "        #print(torch.cat((torch.cat((out,out2),1),out3),1).shape)\n",
    "        final_out = self.fcFinal(torch.cat((torch.cat((out,out2),1),out3),1))\n",
    "        #print(\"hello\")\n",
    "        final_out = self.softmax(final_out)\n",
    "        return final_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalize the data\n",
    "x_train_images_norm_torch = x_train_images_torch/255.0\n",
    "x_test_images_norm_torch = x_test_images_torch/255.0\n",
    "\n",
    "x_train_images_norm_torch_100 = x_train_images_torch_100/255.0\n",
    "x_test_images_norm_torch_100 = x_test_images_torch_100/255.0\n",
    "\n",
    "x_train_images_norm_torch_50 = x_train_images_torch_50/255.0\n",
    "x_test_images_norm_torch_50 = x_test_images_torch_50/255.0\n",
    "# \n",
    "cnn_net = CNN().float().to(device)\n",
    "optim = torch.optim.Adam(cnn_net.parameters(), lr=0.001)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "# \n",
    "n_epoches = 150 # To-Do: need more epoches.\n",
    "batch_size = 200 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0\n",
      "0 200\n",
      "0 400\n",
      "0 600\n",
      "0 800\n",
      "0 1000\n",
      "0 1200\n",
      "0 1400\n",
      "0 1600\n",
      "0 1800\n",
      "0 2000\n",
      "0 2200\n",
      "0 2400\n",
      "0 2600\n",
      "0 2800\n",
      "0 3000\n",
      "0 3200\n",
      "0 3400\n",
      "Epoch 0: train loss: 1.2700066566467285; test loss: 1.2753958702087402\n",
      "Epoch 0: train accuracy: 0.41226096737907764; test accuracy: 0.41732283464566927\n",
      "1 0\n",
      "1 200\n",
      "1 400\n",
      "1 600\n",
      "1 800\n",
      "1 1000\n",
      "1 1200\n",
      "1 1400\n",
      "1 1600\n",
      "1 1800\n",
      "1 2000\n",
      "1 2200\n",
      "1 2400\n",
      "1 2600\n",
      "1 2800\n",
      "1 3000\n",
      "1 3200\n",
      "1 3400\n",
      "Epoch 1: train loss: 1.249767780303955; test loss: 1.2905662059783936\n",
      "Epoch 1: train accuracy: 0.38188976377952755; test accuracy: 0.3689538807649044\n",
      "2 0\n",
      "2 200\n",
      "2 400\n",
      "2 600\n",
      "2 800\n",
      "2 1000\n",
      "2 1200\n",
      "2 1400\n",
      "2 1600\n",
      "2 1800\n",
      "2 2000\n",
      "2 2200\n",
      "2 2400\n",
      "2 2600\n",
      "2 2800\n",
      "2 3000\n",
      "2 3200\n",
      "2 3400\n",
      "Epoch 2: train loss: 1.280522108078003; test loss: 1.2523483037948608\n",
      "Epoch 2: train accuracy: 0.4772215973003375; test accuracy: 0.46231721034870643\n",
      "3 0\n",
      "3 200\n",
      "3 400\n",
      "3 600\n",
      "3 800\n",
      "3 1000\n",
      "3 1200\n",
      "3 1400\n",
      "3 1600\n",
      "3 1800\n",
      "3 2000\n",
      "3 2200\n",
      "3 2400\n",
      "3 2600\n",
      "3 2800\n",
      "3 3000\n",
      "3 3200\n",
      "3 3400\n",
      "Epoch 3: train loss: 1.250611424446106; test loss: 1.253287672996521\n",
      "Epoch 3: train accuracy: 0.4735658042744657; test accuracy: 0.46006749156355453\n",
      "4 0\n",
      "4 200\n",
      "4 400\n",
      "4 600\n",
      "4 800\n",
      "4 1000\n",
      "4 1200\n",
      "4 1400\n",
      "4 1600\n",
      "4 1800\n",
      "4 2000\n",
      "4 2200\n",
      "4 2400\n",
      "4 2600\n",
      "4 2800\n",
      "4 3000\n",
      "4 3200\n",
      "4 3400\n",
      "Epoch 4: train loss: 1.2300870418548584; test loss: 1.2516142129898071\n",
      "Epoch 4: train accuracy: 0.4628796400449944; test accuracy: 0.4409448818897638\n",
      "5 0\n",
      "5 200\n",
      "5 400\n",
      "5 600\n",
      "5 800\n",
      "5 1000\n",
      "5 1200\n",
      "5 1400\n",
      "5 1600\n",
      "5 1800\n",
      "5 2000\n",
      "5 2200\n",
      "5 2400\n",
      "5 2600\n",
      "5 2800\n",
      "5 3000\n",
      "5 3200\n",
      "5 3400\n",
      "Epoch 5: train loss: 1.2363109588623047; test loss: 1.230757713317871\n",
      "Epoch 5: train accuracy: 0.4932508436445444; test accuracy: 0.4881889763779528\n",
      "6 0\n",
      "6 200\n",
      "6 400\n",
      "6 600\n",
      "6 800\n",
      "6 1000\n",
      "6 1200\n",
      "6 1400\n",
      "6 1600\n",
      "6 1800\n",
      "6 2000\n",
      "6 2200\n",
      "6 2400\n",
      "6 2600\n",
      "6 2800\n",
      "6 3000\n",
      "6 3200\n",
      "6 3400\n",
      "Epoch 6: train loss: 1.200422763824463; test loss: 1.236617088317871\n",
      "Epoch 6: train accuracy: 0.49015748031496065; test accuracy: 0.47581552305961755\n",
      "7 0\n",
      "7 200\n",
      "7 400\n",
      "7 600\n",
      "7 800\n",
      "7 1000\n",
      "7 1200\n",
      "7 1400\n",
      "7 1600\n",
      "7 1800\n",
      "7 2000\n",
      "7 2200\n",
      "7 2400\n",
      "7 2600\n",
      "7 2800\n",
      "7 3000\n",
      "7 3200\n",
      "7 3400\n",
      "Epoch 7: train loss: 1.1867992877960205; test loss: 1.19760262966156\n",
      "Epoch 7: train accuracy: 0.5590551181102362; test accuracy: 0.5489313835770528\n",
      "8 0\n",
      "8 200\n",
      "8 400\n",
      "8 600\n",
      "8 800\n",
      "8 1000\n",
      "8 1200\n",
      "8 1400\n",
      "8 1600\n",
      "8 1800\n",
      "8 2000\n",
      "8 2200\n",
      "8 2400\n",
      "8 2600\n",
      "8 2800\n",
      "8 3000\n",
      "8 3200\n",
      "8 3400\n",
      "Epoch 8: train loss: 1.1598421335220337; test loss: 1.1889593601226807\n",
      "Epoch 8: train accuracy: 0.5717097862767154; test accuracy: 0.5489313835770528\n",
      "9 0\n",
      "9 200\n",
      "9 400\n",
      "9 600\n",
      "9 800\n",
      "9 1000\n",
      "9 1200\n",
      "9 1400\n",
      "9 1600\n",
      "9 1800\n",
      "9 2000\n",
      "9 2200\n",
      "9 2400\n",
      "9 2600\n",
      "9 2800\n",
      "9 3000\n",
      "9 3200\n",
      "9 3400\n",
      "Epoch 9: train loss: 1.1488146781921387; test loss: 1.1674224138259888\n",
      "Epoch 9: train accuracy: 0.6034870641169854; test accuracy: 0.5691788526434196\n",
      "10 0\n",
      "10 200\n",
      "10 400\n",
      "10 600\n",
      "10 800\n",
      "10 1000\n",
      "10 1200\n",
      "10 1400\n",
      "10 1600\n",
      "10 1800\n",
      "10 2000\n",
      "10 2200\n",
      "10 2400\n",
      "10 2600\n",
      "10 2800\n",
      "10 3000\n",
      "10 3200\n",
      "10 3400\n",
      "Epoch 10: train loss: 1.1291234493255615; test loss: 1.1754367351531982\n",
      "Epoch 10: train accuracy: 0.5902699662542182; test accuracy: 0.5646794150731158\n",
      "11 0\n",
      "11 200\n",
      "11 400\n",
      "11 600\n",
      "11 800\n",
      "11 1000\n",
      "11 1200\n",
      "11 1400\n",
      "11 1600\n",
      "11 1800\n",
      "11 2000\n",
      "11 2200\n",
      "11 2400\n",
      "11 2600\n",
      "11 2800\n",
      "11 3000\n",
      "11 3200\n",
      "11 3400\n",
      "Epoch 11: train loss: 1.1898819208145142; test loss: 1.1302365064620972\n",
      "Epoch 11: train accuracy: 0.6338582677165354; test accuracy: 0.6029246344206974\n",
      "12 0\n",
      "12 200\n",
      "12 400\n",
      "12 600\n",
      "12 800\n",
      "12 1000\n",
      "12 1200\n",
      "12 1400\n",
      "12 1600\n",
      "12 1800\n",
      "12 2000\n",
      "12 2200\n",
      "12 2400\n",
      "12 2600\n",
      "12 2800\n",
      "12 3000\n",
      "12 3200\n",
      "12 3400\n",
      "Epoch 12: train loss: 1.179230809211731; test loss: 1.1442344188690186\n",
      "Epoch 12: train accuracy: 0.6051743532058492; test accuracy: 0.5883014623172104\n",
      "13 0\n",
      "13 200\n",
      "13 400\n",
      "13 600\n",
      "13 800\n",
      "13 1000\n",
      "13 1200\n",
      "13 1400\n",
      "13 1600\n",
      "13 1800\n",
      "13 2000\n",
      "13 2200\n",
      "13 2400\n",
      "13 2600\n",
      "13 2800\n",
      "13 3000\n",
      "13 3200\n",
      "13 3400\n",
      "Epoch 13: train loss: 1.1209670305252075; test loss: 1.134821891784668\n",
      "Epoch 13: train accuracy: 0.6299212598425197; test accuracy: 0.6062992125984252\n",
      "14 0\n",
      "14 200\n",
      "14 400\n",
      "14 600\n",
      "14 800\n",
      "14 1000\n",
      "14 1200\n",
      "14 1400\n",
      "14 1600\n",
      "14 1800\n",
      "14 2000\n",
      "14 2200\n",
      "14 2400\n",
      "14 2600\n",
      "14 2800\n",
      "14 3000\n",
      "14 3200\n",
      "14 3400\n",
      "Epoch 14: train loss: 1.1087243556976318; test loss: 1.1171464920043945\n",
      "Epoch 14: train accuracy: 0.6456692913385826; test accuracy: 0.6175478065241845\n",
      "15 0\n",
      "15 200\n",
      "15 400\n",
      "15 600\n",
      "15 800\n",
      "15 1000\n",
      "15 1200\n",
      "15 1400\n",
      "15 1600\n",
      "15 1800\n",
      "15 2000\n",
      "15 2200\n",
      "15 2400\n",
      "15 2600\n",
      "15 2800\n",
      "15 3000\n",
      "15 3200\n",
      "15 3400\n",
      "Epoch 15: train loss: 1.1040678024291992; test loss: 1.1146507263183594\n",
      "Epoch 15: train accuracy: 0.6487626546681665; test accuracy: 0.6276715410573678\n",
      "16 0\n",
      "16 200\n",
      "16 400\n",
      "16 600\n",
      "16 800\n",
      "16 1000\n",
      "16 1200\n",
      "16 1400\n",
      "16 1600\n",
      "16 1800\n",
      "16 2000\n",
      "16 2200\n",
      "16 2400\n",
      "16 2600\n",
      "16 2800\n",
      "16 3000\n",
      "16 3200\n",
      "16 3400\n",
      "Epoch 16: train loss: 1.100865364074707; test loss: 1.1049566268920898\n",
      "Epoch 16: train accuracy: 0.655793025871766; test accuracy: 0.6276715410573678\n",
      "17 0\n",
      "17 200\n",
      "17 400\n",
      "17 600\n",
      "17 800\n",
      "17 1000\n",
      "17 1200\n",
      "17 1400\n",
      "17 1600\n",
      "17 1800\n",
      "17 2000\n",
      "17 2200\n",
      "17 2400\n",
      "17 2600\n",
      "17 2800\n",
      "17 3000\n",
      "17 3200\n",
      "17 3400\n",
      "Epoch 17: train loss: 1.1006935834884644; test loss: 1.1196281909942627\n",
      "Epoch 17: train accuracy: 0.640607424071991; test accuracy: 0.6164229471316085\n",
      "18 0\n",
      "18 200\n",
      "18 400\n",
      "18 600\n",
      "18 800\n",
      "18 1000\n",
      "18 1200\n",
      "18 1400\n",
      "18 1600\n",
      "18 1800\n",
      "18 2000\n",
      "18 2200\n",
      "18 2400\n",
      "18 2600\n",
      "18 2800\n",
      "18 3000\n",
      "18 3200\n",
      "18 3400\n",
      "Epoch 18: train loss: 1.0834022760391235; test loss: 1.1053194999694824\n",
      "Epoch 18: train accuracy: 0.6484814398200225; test accuracy: 0.6321709786276716\n",
      "19 0\n",
      "19 200\n",
      "19 400\n",
      "19 600\n",
      "19 800\n",
      "19 1000\n",
      "19 1200\n",
      "19 1400\n",
      "19 1600\n",
      "19 1800\n",
      "19 2000\n",
      "19 2200\n",
      "19 2400\n",
      "19 2600\n",
      "19 2800\n",
      "19 3000\n",
      "19 3200\n",
      "19 3400\n",
      "Epoch 19: train loss: 1.0777812004089355; test loss: 1.1093875169754028\n",
      "Epoch 19: train accuracy: 0.6611361079865017; test accuracy: 0.6310461192350956\n",
      "20 0\n",
      "20 200\n",
      "20 400\n",
      "20 600\n",
      "20 800\n",
      "20 1000\n",
      "20 1200\n",
      "20 1400\n",
      "20 1600\n",
      "20 1800\n",
      "20 2000\n",
      "20 2200\n",
      "20 2400\n",
      "20 2600\n",
      "20 2800\n",
      "20 3000\n",
      "20 3200\n",
      "20 3400\n",
      "Epoch 20: train loss: 1.0853925943374634; test loss: 1.0970871448516846\n",
      "Epoch 20: train accuracy: 0.6664791901012374; test accuracy: 0.6310461192350956\n",
      "21 0\n",
      "21 200\n",
      "21 400\n",
      "21 600\n",
      "21 800\n",
      "21 1000\n",
      "21 1200\n",
      "21 1400\n",
      "21 1600\n",
      "21 1800\n",
      "21 2000\n",
      "21 2200\n",
      "21 2400\n",
      "21 2600\n",
      "21 2800\n",
      "21 3000\n",
      "21 3200\n",
      "21 3400\n",
      "Epoch 21: train loss: 1.0814586877822876; test loss: 1.0971930027008057\n",
      "Epoch 21: train accuracy: 0.6656355455568054; test accuracy: 0.6445444319460067\n",
      "22 0\n",
      "22 200\n",
      "22 400\n",
      "22 600\n",
      "22 800\n",
      "22 1000\n",
      "22 1200\n",
      "22 1400\n",
      "22 1600\n",
      "22 1800\n",
      "22 2000\n",
      "22 2200\n",
      "22 2400\n",
      "22 2600\n",
      "22 2800\n",
      "22 3000\n",
      "22 3200\n",
      "22 3400\n",
      "Epoch 22: train loss: 1.0791661739349365; test loss: 1.0837161540985107\n",
      "Epoch 22: train accuracy: 0.6746344206974129; test accuracy: 0.6580427446569179\n",
      "23 0\n",
      "23 200\n",
      "23 400\n",
      "23 600\n",
      "23 800\n",
      "23 1000\n",
      "23 1200\n",
      "23 1400\n",
      "23 1600\n",
      "23 1800\n",
      "23 2000\n",
      "23 2200\n",
      "23 2400\n",
      "23 2600\n",
      "23 2800\n",
      "23 3000\n",
      "23 3200\n",
      "23 3400\n",
      "Epoch 23: train loss: 1.0815569162368774; test loss: 1.084848165512085\n",
      "Epoch 23: train accuracy: 0.6763217097862767; test accuracy: 0.6524184476940382\n",
      "24 0\n",
      "24 200\n",
      "24 400\n",
      "24 600\n",
      "24 800\n",
      "24 1000\n",
      "24 1200\n",
      "24 1400\n",
      "24 1600\n",
      "24 1800\n",
      "24 2000\n",
      "24 2200\n",
      "24 2400\n",
      "24 2600\n",
      "24 2800\n",
      "24 3000\n",
      "24 3200\n",
      "24 3400\n",
      "Epoch 24: train loss: 1.0767873525619507; test loss: 1.0789074897766113\n",
      "Epoch 24: train accuracy: 0.6827896512935883; test accuracy: 0.6625421822272216\n",
      "25 0\n",
      "25 200\n",
      "25 400\n",
      "25 600\n",
      "25 800\n",
      "25 1000\n",
      "25 1200\n",
      "25 1400\n",
      "25 1600\n",
      "25 1800\n",
      "25 2000\n",
      "25 2200\n",
      "25 2400\n",
      "25 2600\n",
      "25 2800\n",
      "25 3000\n",
      "25 3200\n",
      "25 3400\n",
      "Epoch 25: train loss: 1.0655782222747803; test loss: 1.078611135482788\n",
      "Epoch 25: train accuracy: 0.6847581552305961; test accuracy: 0.6591676040494938\n",
      "26 0\n",
      "26 200\n",
      "26 400\n",
      "26 600\n",
      "26 800\n",
      "26 1000\n",
      "26 1200\n",
      "26 1400\n",
      "26 1600\n",
      "26 1800\n",
      "26 2000\n",
      "26 2200\n",
      "26 2400\n",
      "26 2600\n",
      "26 2800\n",
      "26 3000\n",
      "26 3200\n",
      "26 3400\n",
      "Epoch 26: train loss: 1.0682445764541626; test loss: 1.0746822357177734\n",
      "Epoch 26: train accuracy: 0.6853205849268842; test accuracy: 0.6670416197975253\n",
      "27 0\n",
      "27 200\n",
      "27 400\n",
      "27 600\n",
      "27 800\n",
      "27 1000\n",
      "27 1200\n",
      "27 1400\n",
      "27 1600\n",
      "27 1800\n",
      "27 2000\n",
      "27 2200\n",
      "27 2400\n",
      "27 2600\n",
      "27 2800\n",
      "27 3000\n",
      "27 3200\n",
      "27 3400\n",
      "Epoch 27: train loss: 1.0611414909362793; test loss: 1.0684623718261719\n",
      "Epoch 27: train accuracy: 0.6909448818897638; test accuracy: 0.6704161979752531\n",
      "28 0\n",
      "28 200\n",
      "28 400\n",
      "28 600\n",
      "28 800\n",
      "28 1000\n",
      "28 1200\n",
      "28 1400\n",
      "28 1600\n",
      "28 1800\n",
      "28 2000\n",
      "28 2200\n",
      "28 2400\n",
      "28 2600\n",
      "28 2800\n",
      "28 3000\n",
      "28 3200\n",
      "28 3400\n",
      "Epoch 28: train loss: 1.062183141708374; test loss: 1.0752965211868286\n",
      "Epoch 28: train accuracy: 0.6906636670416197; test accuracy: 0.6602924634420697\n",
      "29 0\n",
      "29 200\n",
      "29 400\n",
      "29 600\n",
      "29 800\n",
      "29 1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29 1200\n",
      "29 1400\n",
      "29 1600\n",
      "29 1800\n",
      "29 2000\n",
      "29 2200\n",
      "29 2400\n",
      "29 2600\n",
      "29 2800\n",
      "29 3000\n",
      "29 3200\n",
      "29 3400\n",
      "Epoch 29: train loss: 1.058440089225769; test loss: 1.0658531188964844\n",
      "Epoch 29: train accuracy: 0.7005061867266592; test accuracy: 0.6794150731158605\n",
      "30 0\n",
      "30 200\n",
      "30 400\n",
      "30 600\n",
      "30 800\n",
      "30 1000\n",
      "30 1200\n",
      "30 1400\n",
      "30 1600\n",
      "30 1800\n",
      "30 2000\n",
      "30 2200\n",
      "30 2400\n",
      "30 2600\n",
      "30 2800\n",
      "30 3000\n",
      "30 3200\n",
      "30 3400\n",
      "Epoch 30: train loss: 1.0464109182357788; test loss: 1.0803872346878052\n",
      "Epoch 30: train accuracy: 0.6979752530933633; test accuracy: 0.6602924634420697\n",
      "31 0\n",
      "31 200\n",
      "31 400\n",
      "31 600\n",
      "31 800\n",
      "31 1000\n",
      "31 1200\n",
      "31 1400\n",
      "31 1600\n",
      "31 1800\n",
      "31 2000\n",
      "31 2200\n",
      "31 2400\n",
      "31 2600\n",
      "31 2800\n",
      "31 3000\n",
      "31 3200\n",
      "31 3400\n",
      "Epoch 31: train loss: 1.0508739948272705; test loss: 1.0598235130310059\n",
      "Epoch 31: train accuracy: 0.7095050618672666; test accuracy: 0.6906636670416197\n",
      "32 0\n",
      "32 200\n",
      "32 400\n",
      "32 600\n",
      "32 800\n",
      "32 1000\n",
      "32 1200\n",
      "32 1400\n",
      "32 1600\n",
      "32 1800\n",
      "32 2000\n",
      "32 2200\n",
      "32 2400\n",
      "32 2600\n",
      "32 2800\n",
      "32 3000\n",
      "32 3200\n",
      "32 3400\n",
      "Epoch 32: train loss: 1.0435504913330078; test loss: 1.0601366758346558\n",
      "Epoch 32: train accuracy: 0.7131608548931384; test accuracy: 0.6906636670416197\n",
      "33 0\n",
      "33 200\n",
      "33 400\n",
      "33 600\n",
      "33 800\n",
      "33 1000\n",
      "33 1200\n",
      "33 1400\n",
      "33 1600\n",
      "33 1800\n",
      "33 2000\n",
      "33 2200\n",
      "33 2400\n",
      "33 2600\n",
      "33 2800\n",
      "33 3000\n",
      "33 3200\n",
      "33 3400\n",
      "Epoch 33: train loss: 1.0466606616973877; test loss: 1.054685115814209\n",
      "Epoch 33: train accuracy: 0.7089426321709786; test accuracy: 0.6917885264341957\n",
      "34 0\n",
      "34 200\n",
      "34 400\n",
      "34 600\n",
      "34 800\n",
      "34 1000\n",
      "34 1200\n",
      "34 1400\n",
      "34 1600\n",
      "34 1800\n",
      "34 2000\n",
      "34 2200\n",
      "34 2400\n",
      "34 2600\n",
      "34 2800\n",
      "34 3000\n",
      "34 3200\n",
      "34 3400\n",
      "Epoch 34: train loss: 1.0466827154159546; test loss: 1.052056074142456\n",
      "Epoch 34: train accuracy: 0.7078177727784027; test accuracy: 0.6906636670416197\n",
      "35 0\n",
      "35 200\n",
      "35 400\n",
      "35 600\n",
      "35 800\n",
      "35 1000\n",
      "35 1200\n",
      "35 1400\n",
      "35 1600\n",
      "35 1800\n",
      "35 2000\n",
      "35 2200\n",
      "35 2400\n",
      "35 2600\n",
      "35 2800\n",
      "35 3000\n",
      "35 3200\n",
      "35 3400\n",
      "Epoch 35: train loss: 1.0491712093353271; test loss: 1.0595415830612183\n",
      "Epoch 35: train accuracy: 0.7016310461192351; test accuracy: 0.688413948256468\n",
      "36 0\n",
      "36 200\n",
      "36 400\n",
      "36 600\n",
      "36 800\n",
      "36 1000\n",
      "36 1200\n",
      "36 1400\n",
      "36 1600\n",
      "36 1800\n",
      "36 2000\n",
      "36 2200\n",
      "36 2400\n",
      "36 2600\n",
      "36 2800\n",
      "36 3000\n",
      "36 3200\n",
      "36 3400\n",
      "Epoch 36: train loss: 1.0427354574203491; test loss: 1.0543575286865234\n",
      "Epoch 36: train accuracy: 0.7170978627671541; test accuracy: 0.688413948256468\n",
      "37 0\n",
      "37 200\n",
      "37 400\n",
      "37 600\n",
      "37 800\n",
      "37 1000\n",
      "37 1200\n",
      "37 1400\n",
      "37 1600\n",
      "37 1800\n",
      "37 2000\n",
      "37 2200\n",
      "37 2400\n",
      "37 2600\n",
      "37 2800\n",
      "37 3000\n",
      "37 3200\n",
      "37 3400\n",
      "Epoch 37: train loss: 1.0403255224227905; test loss: 1.0522185564041138\n",
      "Epoch 37: train accuracy: 0.7151293588301463; test accuracy: 0.6940382452193475\n",
      "38 0\n",
      "38 200\n",
      "38 400\n",
      "38 600\n",
      "38 800\n",
      "38 1000\n",
      "38 1200\n",
      "38 1400\n",
      "38 1600\n",
      "38 1800\n",
      "38 2000\n",
      "38 2200\n",
      "38 2400\n",
      "38 2600\n",
      "38 2800\n",
      "38 3000\n",
      "38 3200\n",
      "38 3400\n",
      "Epoch 38: train loss: 1.0424766540527344; test loss: 1.0491379499435425\n",
      "Epoch 38: train accuracy: 0.7168166479190101; test accuracy: 0.6940382452193475\n",
      "39 0\n",
      "39 200\n",
      "39 400\n",
      "39 600\n",
      "39 800\n",
      "39 1000\n",
      "39 1200\n",
      "39 1400\n",
      "39 1600\n",
      "39 1800\n",
      "39 2000\n",
      "39 2200\n",
      "39 2400\n",
      "39 2600\n",
      "39 2800\n",
      "39 3000\n",
      "39 3200\n",
      "39 3400\n",
      "Epoch 39: train loss: 1.042722225189209; test loss: 1.048313856124878\n",
      "Epoch 39: train accuracy: 0.7168166479190101; test accuracy: 0.6940382452193475\n",
      "40 0\n",
      "40 200\n",
      "40 400\n",
      "40 600\n",
      "40 800\n",
      "40 1000\n",
      "40 1200\n",
      "40 1400\n",
      "40 1600\n",
      "40 1800\n",
      "40 2000\n",
      "40 2200\n",
      "40 2400\n",
      "40 2600\n",
      "40 2800\n",
      "40 3000\n",
      "40 3200\n",
      "40 3400\n",
      "Epoch 40: train loss: 1.0422654151916504; test loss: 1.047907829284668\n",
      "Epoch 40: train accuracy: 0.718503937007874; test accuracy: 0.6940382452193475\n",
      "41 0\n",
      "41 200\n",
      "41 400\n",
      "41 600\n",
      "41 800\n",
      "41 1000\n",
      "41 1200\n",
      "41 1400\n",
      "41 1600\n",
      "41 1800\n",
      "41 2000\n",
      "41 2200\n",
      "41 2400\n",
      "41 2600\n",
      "41 2800\n",
      "41 3000\n",
      "41 3200\n",
      "41 3400\n",
      "Epoch 41: train loss: 1.0435205698013306; test loss: 1.0450267791748047\n",
      "Epoch 41: train accuracy: 0.7207536557930259; test accuracy: 0.6985376827896513\n",
      "42 0\n",
      "42 200\n",
      "42 400\n",
      "42 600\n",
      "42 800\n",
      "42 1000\n",
      "42 1200\n",
      "42 1400\n",
      "42 1600\n",
      "42 1800\n",
      "42 2000\n",
      "42 2200\n",
      "42 2400\n",
      "42 2600\n",
      "42 2800\n",
      "42 3000\n",
      "42 3200\n",
      "42 3400\n",
      "Epoch 42: train loss: 1.0448005199432373; test loss: 1.0446057319641113\n",
      "Epoch 42: train accuracy: 0.7207536557930259; test accuracy: 0.6985376827896513\n",
      "43 0\n",
      "43 200\n",
      "43 400\n",
      "43 600\n",
      "43 800\n",
      "43 1000\n",
      "43 1200\n",
      "43 1400\n",
      "43 1600\n",
      "43 1800\n",
      "43 2000\n",
      "43 2200\n",
      "43 2400\n",
      "43 2600\n",
      "43 2800\n",
      "43 3000\n",
      "43 3200\n",
      "43 3400\n",
      "Epoch 43: train loss: 1.0439019203186035; test loss: 1.044973611831665\n",
      "Epoch 43: train accuracy: 0.7221597300337458; test accuracy: 0.6985376827896513\n",
      "44 0\n",
      "44 200\n",
      "44 400\n",
      "44 600\n",
      "44 800\n",
      "44 1000\n",
      "44 1200\n",
      "44 1400\n",
      "44 1600\n",
      "44 1800\n",
      "44 2000\n",
      "44 2200\n",
      "44 2400\n",
      "44 2600\n",
      "44 2800\n",
      "44 3000\n",
      "44 3200\n",
      "44 3400\n",
      "Epoch 44: train loss: 1.0420806407928467; test loss: 1.0456371307373047\n",
      "Epoch 44: train accuracy: 0.7221597300337458; test accuracy: 0.6985376827896513\n",
      "45 0\n",
      "45 200\n",
      "45 400\n",
      "45 600\n",
      "45 800\n",
      "45 1000\n",
      "45 1200\n",
      "45 1400\n",
      "45 1600\n",
      "45 1800\n",
      "45 2000\n",
      "45 2200\n",
      "45 2400\n",
      "45 2600\n",
      "45 2800\n",
      "45 3000\n",
      "45 3200\n",
      "45 3400\n",
      "Epoch 45: train loss: 1.038784146308899; test loss: 1.0469739437103271\n",
      "Epoch 45: train accuracy: 0.7221597300337458; test accuracy: 0.6940382452193475\n",
      "46 0\n",
      "46 200\n",
      "46 400\n",
      "46 600\n",
      "46 800\n",
      "46 1000\n",
      "46 1200\n",
      "46 1400\n",
      "46 1600\n",
      "46 1800\n",
      "46 2000\n",
      "46 2200\n",
      "46 2400\n",
      "46 2600\n",
      "46 2800\n",
      "46 3000\n",
      "46 3200\n",
      "46 3400\n",
      "Epoch 46: train loss: 1.041400671005249; test loss: 1.0449144840240479\n",
      "Epoch 46: train accuracy: 0.7224409448818898; test accuracy: 0.6962879640044994\n",
      "47 0\n",
      "47 200\n",
      "47 400\n",
      "47 600\n",
      "47 800\n",
      "47 1000\n",
      "47 1200\n",
      "47 1400\n",
      "47 1600\n",
      "47 1800\n",
      "47 2000\n",
      "47 2200\n",
      "47 2400\n",
      "47 2600\n",
      "47 2800\n",
      "47 3000\n",
      "47 3200\n",
      "47 3400\n",
      "Epoch 47: train loss: 1.0413306951522827; test loss: 1.0448596477508545\n",
      "Epoch 47: train accuracy: 0.7224409448818898; test accuracy: 0.6962879640044994\n",
      "48 0\n",
      "48 200\n",
      "48 400\n",
      "48 600\n",
      "48 800\n",
      "48 1000\n",
      "48 1200\n",
      "48 1400\n",
      "48 1600\n",
      "48 1800\n",
      "48 2000\n",
      "48 2200\n",
      "48 2400\n",
      "48 2600\n",
      "48 2800\n",
      "48 3000\n",
      "48 3200\n",
      "48 3400\n",
      "Epoch 48: train loss: 1.041150450706482; test loss: 1.0455695390701294\n",
      "Epoch 48: train accuracy: 0.7224409448818898; test accuracy: 0.6962879640044994\n",
      "49 0\n",
      "49 200\n",
      "49 400\n",
      "49 600\n",
      "49 800\n",
      "49 1000\n",
      "49 1200\n",
      "49 1400\n",
      "49 1600\n",
      "49 1800\n",
      "49 2000\n",
      "49 2200\n",
      "49 2400\n",
      "49 2600\n",
      "49 2800\n",
      "49 3000\n",
      "49 3200\n",
      "49 3400\n",
      "Epoch 49: train loss: 1.0432485342025757; test loss: 1.045358657836914\n",
      "Epoch 49: train accuracy: 0.7241282339707537; test accuracy: 0.6996625421822272\n",
      "50 0\n",
      "50 200\n",
      "50 400\n",
      "50 600\n",
      "50 800\n",
      "50 1000\n",
      "50 1200\n",
      "50 1400\n",
      "50 1600\n",
      "50 1800\n",
      "50 2000\n",
      "50 2200\n",
      "50 2400\n",
      "50 2600\n",
      "50 2800\n",
      "50 3000\n",
      "50 3200\n",
      "50 3400\n",
      "Epoch 50: train loss: 1.0405325889587402; test loss: 1.0462278127670288\n",
      "Epoch 50: train accuracy: 0.7241282339707537; test accuracy: 0.6962879640044994\n",
      "51 0\n",
      "51 200\n",
      "51 400\n",
      "51 600\n",
      "51 800\n",
      "51 1000\n",
      "51 1200\n",
      "51 1400\n",
      "51 1600\n",
      "51 1800\n",
      "51 2000\n",
      "51 2200\n",
      "51 2400\n",
      "51 2600\n",
      "51 2800\n",
      "51 3000\n",
      "51 3200\n",
      "51 3400\n",
      "Epoch 51: train loss: 1.0391303300857544; test loss: 1.0476329326629639\n",
      "Epoch 51: train accuracy: 0.7241282339707537; test accuracy: 0.6951631046119235\n",
      "52 0\n",
      "52 200\n",
      "52 400\n",
      "52 600\n",
      "52 800\n",
      "52 1000\n",
      "52 1200\n",
      "52 1400\n",
      "52 1600\n",
      "52 1800\n",
      "52 2000\n",
      "52 2200\n",
      "52 2400\n",
      "52 2600\n",
      "52 2800\n",
      "52 3000\n",
      "52 3200\n",
      "52 3400\n",
      "Epoch 52: train loss: 1.0440804958343506; test loss: 1.1056679487228394\n",
      "Epoch 52: train accuracy: 0.671822272215973; test accuracy: 0.6355455568053994\n",
      "53 0\n",
      "53 200\n",
      "53 400\n",
      "53 600\n",
      "53 800\n",
      "53 1000\n",
      "53 1200\n",
      "53 1400\n",
      "53 1600\n",
      "53 1800\n",
      "53 2000\n",
      "53 2200\n",
      "53 2400\n",
      "53 2600\n",
      "53 2800\n",
      "53 3000\n",
      "53 3200\n",
      "53 3400\n",
      "Epoch 53: train loss: 1.0410739183425903; test loss: 1.048999309539795\n",
      "Epoch 53: train accuracy: 0.7238470191226096; test accuracy: 0.6951631046119235\n",
      "54 0\n",
      "54 200\n",
      "54 400\n",
      "54 600\n",
      "54 800\n",
      "54 1000\n",
      "54 1200\n",
      "54 1400\n",
      "54 1600\n",
      "54 1800\n",
      "54 2000\n",
      "54 2200\n",
      "54 2400\n",
      "54 2600\n",
      "54 2800\n",
      "54 3000\n",
      "54 3200\n",
      "54 3400\n",
      "Epoch 54: train loss: 1.0462566614151; test loss: 1.0512564182281494\n",
      "Epoch 54: train accuracy: 0.7232845894263217; test accuracy: 0.6929133858267716\n",
      "55 0\n",
      "55 200\n",
      "55 400\n",
      "55 600\n",
      "55 800\n",
      "55 1000\n",
      "55 1200\n",
      "55 1400\n",
      "55 1600\n",
      "55 1800\n",
      "55 2000\n",
      "55 2200\n",
      "55 2400\n",
      "55 2600\n",
      "55 2800\n",
      "55 3000\n",
      "55 3200\n",
      "55 3400\n",
      "Epoch 55: train loss: 1.0399348735809326; test loss: 1.0458691120147705\n",
      "Epoch 55: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "56 0\n",
      "56 200\n",
      "56 400\n",
      "56 600\n",
      "56 800\n",
      "56 1000\n",
      "56 1200\n",
      "56 1400\n",
      "56 1600\n",
      "56 1800\n",
      "56 2000\n",
      "56 2200\n",
      "56 2400\n",
      "56 2600\n",
      "56 2800\n",
      "56 3000\n",
      "56 3200\n",
      "56 3400\n",
      "Epoch 56: train loss: 1.0386910438537598; test loss: 1.0456410646438599\n",
      "Epoch 56: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "57 0\n",
      "57 200\n",
      "57 400\n",
      "57 600\n",
      "57 800\n",
      "57 1000\n",
      "57 1200\n",
      "57 1400\n",
      "57 1600\n",
      "57 1800\n",
      "57 2000\n",
      "57 2200\n",
      "57 2400\n",
      "57 2600\n",
      "57 2800\n",
      "57 3000\n",
      "57 3200\n",
      "57 3400\n",
      "Epoch 57: train loss: 1.0390607118606567; test loss: 1.0452525615692139\n",
      "Epoch 57: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "58 0\n",
      "58 200\n",
      "58 400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58 600\n",
      "58 800\n",
      "58 1000\n",
      "58 1200\n",
      "58 1400\n",
      "58 1600\n",
      "58 1800\n",
      "58 2000\n",
      "58 2200\n",
      "58 2400\n",
      "58 2600\n",
      "58 2800\n",
      "58 3000\n",
      "58 3200\n",
      "58 3400\n",
      "Epoch 58: train loss: 1.0389894247055054; test loss: 1.0451016426086426\n",
      "Epoch 58: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "59 0\n",
      "59 200\n",
      "59 400\n",
      "59 600\n",
      "59 800\n",
      "59 1000\n",
      "59 1200\n",
      "59 1400\n",
      "59 1600\n",
      "59 1800\n",
      "59 2000\n",
      "59 2200\n",
      "59 2400\n",
      "59 2600\n",
      "59 2800\n",
      "59 3000\n",
      "59 3200\n",
      "59 3400\n",
      "Epoch 59: train loss: 1.03915274143219; test loss: 1.044906735420227\n",
      "Epoch 59: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "60 0\n",
      "60 200\n",
      "60 400\n",
      "60 600\n",
      "60 800\n",
      "60 1000\n",
      "60 1200\n",
      "60 1400\n",
      "60 1600\n",
      "60 1800\n",
      "60 2000\n",
      "60 2200\n",
      "60 2400\n",
      "60 2600\n",
      "60 2800\n",
      "60 3000\n",
      "60 3200\n",
      "60 3400\n",
      "Epoch 60: train loss: 1.0395113229751587; test loss: 1.0447837114334106\n",
      "Epoch 60: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "61 0\n",
      "61 200\n",
      "61 400\n",
      "61 600\n",
      "61 800\n",
      "61 1000\n",
      "61 1200\n",
      "61 1400\n",
      "61 1600\n",
      "61 1800\n",
      "61 2000\n",
      "61 2200\n",
      "61 2400\n",
      "61 2600\n",
      "61 2800\n",
      "61 3000\n",
      "61 3200\n",
      "61 3400\n",
      "Epoch 61: train loss: 1.0400264263153076; test loss: 1.0443832874298096\n",
      "Epoch 61: train accuracy: 0.7258155230596175; test accuracy: 0.6974128233970753\n",
      "62 0\n",
      "62 200\n",
      "62 400\n",
      "62 600\n",
      "62 800\n",
      "62 1000\n",
      "62 1200\n",
      "62 1400\n",
      "62 1600\n",
      "62 1800\n",
      "62 2000\n",
      "62 2200\n",
      "62 2400\n",
      "62 2600\n",
      "62 2800\n",
      "62 3000\n",
      "62 3200\n",
      "62 3400\n",
      "Epoch 62: train loss: 1.0401957035064697; test loss: 1.0442198514938354\n",
      "Epoch 62: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "63 0\n",
      "63 200\n",
      "63 400\n",
      "63 600\n",
      "63 800\n",
      "63 1000\n",
      "63 1200\n",
      "63 1400\n",
      "63 1600\n",
      "63 1800\n",
      "63 2000\n",
      "63 2200\n",
      "63 2400\n",
      "63 2600\n",
      "63 2800\n",
      "63 3000\n",
      "63 3200\n",
      "63 3400\n",
      "Epoch 63: train loss: 1.0399188995361328; test loss: 1.0443475246429443\n",
      "Epoch 63: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "64 0\n",
      "64 200\n",
      "64 400\n",
      "64 600\n",
      "64 800\n",
      "64 1000\n",
      "64 1200\n",
      "64 1400\n",
      "64 1600\n",
      "64 1800\n",
      "64 2000\n",
      "64 2200\n",
      "64 2400\n",
      "64 2600\n",
      "64 2800\n",
      "64 3000\n",
      "64 3200\n",
      "64 3400\n",
      "Epoch 64: train loss: 1.0396970510482788; test loss: 1.044371485710144\n",
      "Epoch 64: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "65 0\n",
      "65 200\n",
      "65 400\n",
      "65 600\n",
      "65 800\n",
      "65 1000\n",
      "65 1200\n",
      "65 1400\n",
      "65 1600\n",
      "65 1800\n",
      "65 2000\n",
      "65 2200\n",
      "65 2400\n",
      "65 2600\n",
      "65 2800\n",
      "65 3000\n",
      "65 3200\n",
      "65 3400\n",
      "Epoch 65: train loss: 1.0396857261657715; test loss: 1.044460654258728\n",
      "Epoch 65: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "66 0\n",
      "66 200\n",
      "66 400\n",
      "66 600\n",
      "66 800\n",
      "66 1000\n",
      "66 1200\n",
      "66 1400\n",
      "66 1600\n",
      "66 1800\n",
      "66 2000\n",
      "66 2200\n",
      "66 2400\n",
      "66 2600\n",
      "66 2800\n",
      "66 3000\n",
      "66 3200\n",
      "66 3400\n",
      "Epoch 66: train loss: 1.039703130722046; test loss: 1.044268250465393\n",
      "Epoch 66: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "67 0\n",
      "67 200\n",
      "67 400\n",
      "67 600\n",
      "67 800\n",
      "67 1000\n",
      "67 1200\n",
      "67 1400\n",
      "67 1600\n",
      "67 1800\n",
      "67 2000\n",
      "67 2200\n",
      "67 2400\n",
      "67 2600\n",
      "67 2800\n",
      "67 3000\n",
      "67 3200\n",
      "67 3400\n",
      "Epoch 67: train loss: 1.0402778387069702; test loss: 1.04399836063385\n",
      "Epoch 67: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "68 0\n",
      "68 200\n",
      "68 400\n",
      "68 600\n",
      "68 800\n",
      "68 1000\n",
      "68 1200\n",
      "68 1400\n",
      "68 1600\n",
      "68 1800\n",
      "68 2000\n",
      "68 2200\n",
      "68 2400\n",
      "68 2600\n",
      "68 2800\n",
      "68 3000\n",
      "68 3200\n",
      "68 3400\n",
      "Epoch 68: train loss: 1.0395004749298096; test loss: 1.0444295406341553\n",
      "Epoch 68: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "69 0\n",
      "69 200\n",
      "69 400\n",
      "69 600\n",
      "69 800\n",
      "69 1000\n",
      "69 1200\n",
      "69 1400\n",
      "69 1600\n",
      "69 1800\n",
      "69 2000\n",
      "69 2200\n",
      "69 2400\n",
      "69 2600\n",
      "69 2800\n",
      "69 3000\n",
      "69 3200\n",
      "69 3400\n",
      "Epoch 69: train loss: 1.0397025346755981; test loss: 1.044602870941162\n",
      "Epoch 69: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "70 0\n",
      "70 200\n",
      "70 400\n",
      "70 600\n",
      "70 800\n",
      "70 1000\n",
      "70 1200\n",
      "70 1400\n",
      "70 1600\n",
      "70 1800\n",
      "70 2000\n",
      "70 2200\n",
      "70 2400\n",
      "70 2600\n",
      "70 2800\n",
      "70 3000\n",
      "70 3200\n",
      "70 3400\n",
      "Epoch 70: train loss: 1.0407094955444336; test loss: 1.0442266464233398\n",
      "Epoch 70: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "71 0\n",
      "71 200\n",
      "71 400\n",
      "71 600\n",
      "71 800\n",
      "71 1000\n",
      "71 1200\n",
      "71 1400\n",
      "71 1600\n",
      "71 1800\n",
      "71 2000\n",
      "71 2200\n",
      "71 2400\n",
      "71 2600\n",
      "71 2800\n",
      "71 3000\n",
      "71 3200\n",
      "71 3400\n",
      "Epoch 71: train loss: 1.0404093265533447; test loss: 1.0443767309188843\n",
      "Epoch 71: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "72 0\n",
      "72 200\n",
      "72 400\n",
      "72 600\n",
      "72 800\n",
      "72 1000\n",
      "72 1200\n",
      "72 1400\n",
      "72 1600\n",
      "72 1800\n",
      "72 2000\n",
      "72 2200\n",
      "72 2400\n",
      "72 2600\n",
      "72 2800\n",
      "72 3000\n",
      "72 3200\n",
      "72 3400\n",
      "Epoch 72: train loss: 1.0402978658676147; test loss: 1.0440491437911987\n",
      "Epoch 72: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "73 0\n",
      "73 200\n",
      "73 400\n",
      "73 600\n",
      "73 800\n",
      "73 1000\n",
      "73 1200\n",
      "73 1400\n",
      "73 1600\n",
      "73 1800\n",
      "73 2000\n",
      "73 2200\n",
      "73 2400\n",
      "73 2600\n",
      "73 2800\n",
      "73 3000\n",
      "73 3200\n",
      "73 3400\n",
      "Epoch 73: train loss: 1.0400738716125488; test loss: 1.0440715551376343\n",
      "Epoch 73: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "74 0\n",
      "74 200\n",
      "74 400\n",
      "74 600\n",
      "74 800\n",
      "74 1000\n",
      "74 1200\n",
      "74 1400\n",
      "74 1600\n",
      "74 1800\n",
      "74 2000\n",
      "74 2200\n",
      "74 2400\n",
      "74 2600\n",
      "74 2800\n",
      "74 3000\n",
      "74 3200\n",
      "74 3400\n",
      "Epoch 74: train loss: 1.0403218269348145; test loss: 1.0442225933074951\n",
      "Epoch 74: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "75 0\n",
      "75 200\n",
      "75 400\n",
      "75 600\n",
      "75 800\n",
      "75 1000\n",
      "75 1200\n",
      "75 1400\n",
      "75 1600\n",
      "75 1800\n",
      "75 2000\n",
      "75 2200\n",
      "75 2400\n",
      "75 2600\n",
      "75 2800\n",
      "75 3000\n",
      "75 3200\n",
      "75 3400\n",
      "Epoch 75: train loss: 1.0402246713638306; test loss: 1.0439990758895874\n",
      "Epoch 75: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "76 0\n",
      "76 200\n",
      "76 400\n",
      "76 600\n",
      "76 800\n",
      "76 1000\n",
      "76 1200\n",
      "76 1400\n",
      "76 1600\n",
      "76 1800\n",
      "76 2000\n",
      "76 2200\n",
      "76 2400\n",
      "76 2600\n",
      "76 2800\n",
      "76 3000\n",
      "76 3200\n",
      "76 3400\n",
      "Epoch 76: train loss: 1.040653944015503; test loss: 1.0435186624526978\n",
      "Epoch 76: train accuracy: 0.7263779527559056; test accuracy: 0.6974128233970753\n",
      "77 0\n",
      "77 200\n",
      "77 400\n",
      "77 600\n",
      "77 800\n",
      "77 1000\n",
      "77 1200\n",
      "77 1400\n",
      "77 1600\n",
      "77 1800\n",
      "77 2000\n",
      "77 2200\n",
      "77 2400\n",
      "77 2600\n",
      "77 2800\n",
      "77 3000\n",
      "77 3200\n",
      "77 3400\n",
      "Epoch 77: train loss: 1.0401664972305298; test loss: 1.0430629253387451\n",
      "Epoch 77: train accuracy: 0.7269403824521935; test accuracy: 0.6996625421822272\n",
      "78 0\n",
      "78 200\n",
      "78 400\n",
      "78 600\n",
      "78 800\n",
      "78 1000\n",
      "78 1200\n",
      "78 1400\n",
      "78 1600\n",
      "78 1800\n",
      "78 2000\n",
      "78 2200\n",
      "78 2400\n",
      "78 2600\n",
      "78 2800\n",
      "78 3000\n",
      "78 3200\n",
      "78 3400\n",
      "Epoch 78: train loss: 1.039898157119751; test loss: 1.042709231376648\n",
      "Epoch 78: train accuracy: 0.7269403824521935; test accuracy: 0.6996625421822272\n",
      "79 0\n",
      "79 200\n",
      "79 400\n",
      "79 600\n",
      "79 800\n",
      "79 1000\n",
      "79 1200\n",
      "79 1400\n",
      "79 1600\n",
      "79 1800\n",
      "79 2000\n",
      "79 2200\n",
      "79 2400\n",
      "79 2600\n",
      "79 2800\n",
      "79 3000\n",
      "79 3200\n",
      "79 3400\n",
      "Epoch 79: train loss: 1.0403634309768677; test loss: 1.0419524908065796\n",
      "Epoch 79: train accuracy: 0.7272215973003374; test accuracy: 0.7007874015748031\n",
      "80 0\n",
      "80 200\n",
      "80 400\n",
      "80 600\n",
      "80 800\n",
      "80 1000\n",
      "80 1200\n",
      "80 1400\n",
      "80 1600\n",
      "80 1800\n",
      "80 2000\n",
      "80 2200\n",
      "80 2400\n",
      "80 2600\n",
      "80 2800\n",
      "80 3000\n",
      "80 3200\n",
      "80 3400\n",
      "Epoch 80: train loss: 1.040130853652954; test loss: 1.0405104160308838\n",
      "Epoch 80: train accuracy: 0.7280652418447694; test accuracy: 0.7007874015748031\n",
      "81 0\n",
      "81 200\n",
      "81 400\n",
      "81 600\n",
      "81 800\n",
      "81 1000\n",
      "81 1200\n",
      "81 1400\n",
      "81 1600\n",
      "81 1800\n",
      "81 2000\n",
      "81 2200\n",
      "81 2400\n",
      "81 2600\n",
      "81 2800\n",
      "81 3000\n",
      "81 3200\n",
      "81 3400\n",
      "Epoch 81: train loss: 1.0402171611785889; test loss: 1.037782907485962\n",
      "Epoch 81: train accuracy: 0.7286276715410573; test accuracy: 0.703037120359955\n",
      "82 0\n",
      "82 200\n",
      "82 400\n",
      "82 600\n",
      "82 800\n",
      "82 1000\n",
      "82 1200\n",
      "82 1400\n",
      "82 1600\n",
      "82 1800\n",
      "82 2000\n",
      "82 2200\n",
      "82 2400\n",
      "82 2600\n",
      "82 2800\n",
      "82 3000\n",
      "82 3200\n",
      "82 3400\n",
      "Epoch 82: train loss: 1.0407342910766602; test loss: 1.037426233291626\n",
      "Epoch 82: train accuracy: 0.7303149606299213; test accuracy: 0.7064116985376828\n",
      "83 0\n",
      "83 200\n",
      "83 400\n",
      "83 600\n",
      "83 800\n",
      "83 1000\n",
      "83 1200\n",
      "83 1400\n",
      "83 1600\n",
      "83 1800\n",
      "83 2000\n",
      "83 2200\n",
      "83 2400\n",
      "83 2600\n",
      "83 2800\n",
      "83 3000\n",
      "83 3200\n",
      "83 3400\n",
      "Epoch 83: train loss: 1.0426442623138428; test loss: 1.0364947319030762\n",
      "Epoch 83: train accuracy: 0.7334083239595051; test accuracy: 0.7097862767154106\n",
      "84 0\n",
      "84 200\n",
      "84 400\n",
      "84 600\n",
      "84 800\n",
      "84 1000\n",
      "84 1200\n",
      "84 1400\n",
      "84 1600\n",
      "84 1800\n",
      "84 2000\n",
      "84 2200\n",
      "84 2400\n",
      "84 2600\n",
      "84 2800\n",
      "84 3000\n",
      "84 3200\n",
      "84 3400\n",
      "Epoch 84: train loss: 1.0458076000213623; test loss: 1.0372027158737183\n",
      "Epoch 84: train accuracy: 0.7314398200224972; test accuracy: 0.7064116985376828\n",
      "85 0\n",
      "85 200\n",
      "85 400\n",
      "85 600\n",
      "85 800\n",
      "85 1000\n",
      "85 1200\n",
      "85 1400\n",
      "85 1600\n",
      "85 1800\n",
      "85 2000\n",
      "85 2200\n",
      "85 2400\n",
      "85 2600\n",
      "85 2800\n",
      "85 3000\n",
      "85 3200\n",
      "85 3400\n",
      "Epoch 85: train loss: 1.053329348564148; test loss: 1.038780927658081\n",
      "Epoch 85: train accuracy: 0.7241282339707537; test accuracy: 0.7052868391451068\n",
      "86 0\n",
      "86 200\n",
      "86 400\n",
      "86 600\n",
      "86 800\n",
      "86 1000\n",
      "86 1200\n",
      "86 1400\n",
      "86 1600\n",
      "86 1800\n",
      "86 2000\n",
      "86 2200\n",
      "86 2400\n",
      "86 2600\n",
      "86 2800\n",
      "86 3000\n",
      "86 3200\n",
      "86 3400\n",
      "Epoch 86: train loss: 1.0455031394958496; test loss: 1.0316908359527588\n",
      "Epoch 86: train accuracy: 0.734533183352081; test accuracy: 0.7131608548931384\n",
      "87 0\n",
      "87 200\n",
      "87 400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 600\n",
      "87 800\n",
      "87 1000\n",
      "87 1200\n",
      "87 1400\n",
      "87 1600\n",
      "87 1800\n",
      "87 2000\n",
      "87 2200\n",
      "87 2400\n",
      "87 2600\n",
      "87 2800\n",
      "87 3000\n",
      "87 3200\n",
      "87 3400\n",
      "Epoch 87: train loss: 1.048600196838379; test loss: 1.033402919769287\n",
      "Epoch 87: train accuracy: 0.734814398200225; test accuracy: 0.7120359955005624\n",
      "88 0\n",
      "88 200\n",
      "88 400\n",
      "88 600\n",
      "88 800\n",
      "88 1000\n",
      "88 1200\n",
      "88 1400\n",
      "88 1600\n",
      "88 1800\n",
      "88 2000\n",
      "88 2200\n",
      "88 2400\n",
      "88 2600\n",
      "88 2800\n",
      "88 3000\n",
      "88 3200\n",
      "88 3400\n",
      "Epoch 88: train loss: 1.051087498664856; test loss: 1.0317089557647705\n",
      "Epoch 88: train accuracy: 0.7365016872890888; test accuracy: 0.7131608548931384\n",
      "89 0\n",
      "89 200\n",
      "89 400\n",
      "89 600\n",
      "89 800\n",
      "89 1000\n",
      "89 1200\n",
      "89 1400\n",
      "89 1600\n",
      "89 1800\n",
      "89 2000\n",
      "89 2200\n",
      "89 2400\n",
      "89 2600\n",
      "89 2800\n",
      "89 3000\n",
      "89 3200\n",
      "89 3400\n",
      "Epoch 89: train loss: 1.0452545881271362; test loss: 1.029950737953186\n",
      "Epoch 89: train accuracy: 0.7373453318335208; test accuracy: 0.7142857142857143\n",
      "90 0\n",
      "90 200\n",
      "90 400\n",
      "90 600\n",
      "90 800\n",
      "90 1000\n",
      "90 1200\n",
      "90 1400\n",
      "90 1600\n",
      "90 1800\n",
      "90 2000\n",
      "90 2200\n",
      "90 2400\n",
      "90 2600\n",
      "90 2800\n",
      "90 3000\n",
      "90 3200\n",
      "90 3400\n",
      "Epoch 90: train loss: 1.0451291799545288; test loss: 1.0293760299682617\n",
      "Epoch 90: train accuracy: 0.7373453318335208; test accuracy: 0.7154105736782902\n",
      "91 0\n",
      "91 200\n",
      "91 400\n",
      "91 600\n",
      "91 800\n",
      "91 1000\n",
      "91 1200\n",
      "91 1400\n",
      "91 1600\n",
      "91 1800\n",
      "91 2000\n",
      "91 2200\n",
      "91 2400\n",
      "91 2600\n",
      "91 2800\n",
      "91 3000\n",
      "91 3200\n",
      "91 3400\n",
      "Epoch 91: train loss: 1.0454192161560059; test loss: 1.030022382736206\n",
      "Epoch 91: train accuracy: 0.7373453318335208; test accuracy: 0.7142857142857143\n",
      "92 0\n",
      "92 200\n",
      "92 400\n",
      "92 600\n",
      "92 800\n",
      "92 1000\n",
      "92 1200\n",
      "92 1400\n",
      "92 1600\n",
      "92 1800\n",
      "92 2000\n",
      "92 2200\n",
      "92 2400\n",
      "92 2600\n",
      "92 2800\n",
      "92 3000\n",
      "92 3200\n",
      "92 3400\n",
      "Epoch 92: train loss: 1.045359492301941; test loss: 1.0324591398239136\n",
      "Epoch 92: train accuracy: 0.7379077615298087; test accuracy: 0.7109111361079865\n",
      "93 0\n",
      "93 200\n",
      "93 400\n",
      "93 600\n",
      "93 800\n",
      "93 1000\n",
      "93 1200\n",
      "93 1400\n",
      "93 1600\n",
      "93 1800\n",
      "93 2000\n",
      "93 2200\n",
      "93 2400\n",
      "93 2600\n",
      "93 2800\n",
      "93 3000\n",
      "93 3200\n",
      "93 3400\n",
      "Epoch 93: train loss: 1.045210599899292; test loss: 1.0327085256576538\n",
      "Epoch 93: train accuracy: 0.7379077615298087; test accuracy: 0.7109111361079865\n",
      "94 0\n",
      "94 200\n",
      "94 400\n",
      "94 600\n",
      "94 800\n",
      "94 1000\n",
      "94 1200\n",
      "94 1400\n",
      "94 1600\n",
      "94 1800\n",
      "94 2000\n",
      "94 2200\n",
      "94 2400\n",
      "94 2600\n",
      "94 2800\n",
      "94 3000\n",
      "94 3200\n",
      "94 3400\n",
      "Epoch 94: train loss: 1.0449717044830322; test loss: 1.0326179265975952\n",
      "Epoch 94: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "95 0\n",
      "95 200\n",
      "95 400\n",
      "95 600\n",
      "95 800\n",
      "95 1000\n",
      "95 1200\n",
      "95 1400\n",
      "95 1600\n",
      "95 1800\n",
      "95 2000\n",
      "95 2200\n",
      "95 2400\n",
      "95 2600\n",
      "95 2800\n",
      "95 3000\n",
      "95 3200\n",
      "95 3400\n",
      "Epoch 95: train loss: 1.0453852415084839; test loss: 1.0329376459121704\n",
      "Epoch 95: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "96 0\n",
      "96 200\n",
      "96 400\n",
      "96 600\n",
      "96 800\n",
      "96 1000\n",
      "96 1200\n",
      "96 1400\n",
      "96 1600\n",
      "96 1800\n",
      "96 2000\n",
      "96 2200\n",
      "96 2400\n",
      "96 2600\n",
      "96 2800\n",
      "96 3000\n",
      "96 3200\n",
      "96 3400\n",
      "Epoch 96: train loss: 1.0451139211654663; test loss: 1.0325406789779663\n",
      "Epoch 96: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "97 0\n",
      "97 200\n",
      "97 400\n",
      "97 600\n",
      "97 800\n",
      "97 1000\n",
      "97 1200\n",
      "97 1400\n",
      "97 1600\n",
      "97 1800\n",
      "97 2000\n",
      "97 2200\n",
      "97 2400\n",
      "97 2600\n",
      "97 2800\n",
      "97 3000\n",
      "97 3200\n",
      "97 3400\n",
      "Epoch 97: train loss: 1.0442067384719849; test loss: 1.0323894023895264\n",
      "Epoch 97: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "98 0\n",
      "98 200\n",
      "98 400\n",
      "98 600\n",
      "98 800\n",
      "98 1000\n",
      "98 1200\n",
      "98 1400\n",
      "98 1600\n",
      "98 1800\n",
      "98 2000\n",
      "98 2200\n",
      "98 2400\n",
      "98 2600\n",
      "98 2800\n",
      "98 3000\n",
      "98 3200\n",
      "98 3400\n",
      "Epoch 98: train loss: 1.0446878671646118; test loss: 1.0324079990386963\n",
      "Epoch 98: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "99 0\n",
      "99 200\n",
      "99 400\n",
      "99 600\n",
      "99 800\n",
      "99 1000\n",
      "99 1200\n",
      "99 1400\n",
      "99 1600\n",
      "99 1800\n",
      "99 2000\n",
      "99 2200\n",
      "99 2400\n",
      "99 2600\n",
      "99 2800\n",
      "99 3000\n",
      "99 3200\n",
      "99 3400\n",
      "Epoch 99: train loss: 1.0446672439575195; test loss: 1.032379150390625\n",
      "Epoch 99: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "100 0\n",
      "100 200\n",
      "100 400\n",
      "100 600\n",
      "100 800\n",
      "100 1000\n",
      "100 1200\n",
      "100 1400\n",
      "100 1600\n",
      "100 1800\n",
      "100 2000\n",
      "100 2200\n",
      "100 2400\n",
      "100 2600\n",
      "100 2800\n",
      "100 3000\n",
      "100 3200\n",
      "100 3400\n",
      "Epoch 100: train loss: 1.045128345489502; test loss: 1.0323041677474976\n",
      "Epoch 100: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "101 0\n",
      "101 200\n",
      "101 400\n",
      "101 600\n",
      "101 800\n",
      "101 1000\n",
      "101 1200\n",
      "101 1400\n",
      "101 1600\n",
      "101 1800\n",
      "101 2000\n",
      "101 2200\n",
      "101 2400\n",
      "101 2600\n",
      "101 2800\n",
      "101 3000\n",
      "101 3200\n",
      "101 3400\n",
      "Epoch 101: train loss: 1.045107364654541; test loss: 1.032424807548523\n",
      "Epoch 101: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "102 0\n",
      "102 200\n",
      "102 400\n",
      "102 600\n",
      "102 800\n",
      "102 1000\n",
      "102 1200\n",
      "102 1400\n",
      "102 1600\n",
      "102 1800\n",
      "102 2000\n",
      "102 2200\n",
      "102 2400\n",
      "102 2600\n",
      "102 2800\n",
      "102 3000\n",
      "102 3200\n",
      "102 3400\n",
      "Epoch 102: train loss: 1.0452219247817993; test loss: 1.032185673713684\n",
      "Epoch 102: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "103 0\n",
      "103 200\n",
      "103 400\n",
      "103 600\n",
      "103 800\n",
      "103 1000\n",
      "103 1200\n",
      "103 1400\n",
      "103 1600\n",
      "103 1800\n",
      "103 2000\n",
      "103 2200\n",
      "103 2400\n",
      "103 2600\n",
      "103 2800\n",
      "103 3000\n",
      "103 3200\n",
      "103 3400\n",
      "Epoch 103: train loss: 1.04515540599823; test loss: 1.032210111618042\n",
      "Epoch 103: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "104 0\n",
      "104 200\n",
      "104 400\n",
      "104 600\n",
      "104 800\n",
      "104 1000\n",
      "104 1200\n",
      "104 1400\n",
      "104 1600\n",
      "104 1800\n",
      "104 2000\n",
      "104 2200\n",
      "104 2400\n",
      "104 2600\n",
      "104 2800\n",
      "104 3000\n",
      "104 3200\n",
      "104 3400\n",
      "Epoch 104: train loss: 1.0452475547790527; test loss: 1.031974196434021\n",
      "Epoch 104: train accuracy: 0.7395950506186727; test accuracy: 0.7120359955005624\n",
      "105 0\n",
      "105 200\n",
      "105 400\n",
      "105 600\n",
      "105 800\n",
      "105 1000\n",
      "105 1200\n",
      "105 1400\n",
      "105 1600\n",
      "105 1800\n",
      "105 2000\n",
      "105 2200\n",
      "105 2400\n",
      "105 2600\n",
      "105 2800\n",
      "105 3000\n",
      "105 3200\n",
      "105 3400\n",
      "Epoch 105: train loss: 1.045064091682434; test loss: 1.0319149494171143\n",
      "Epoch 105: train accuracy: 0.7395950506186727; test accuracy: 0.7120359955005624\n",
      "106 0\n",
      "106 200\n",
      "106 400\n",
      "106 600\n",
      "106 800\n",
      "106 1000\n",
      "106 1200\n",
      "106 1400\n",
      "106 1600\n",
      "106 1800\n",
      "106 2000\n",
      "106 2200\n",
      "106 2400\n",
      "106 2600\n",
      "106 2800\n",
      "106 3000\n",
      "106 3200\n",
      "106 3400\n",
      "Epoch 106: train loss: 1.0451210737228394; test loss: 1.0322023630142212\n",
      "Epoch 106: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "107 0\n",
      "107 200\n",
      "107 400\n",
      "107 600\n",
      "107 800\n",
      "107 1000\n",
      "107 1200\n",
      "107 1400\n",
      "107 1600\n",
      "107 1800\n",
      "107 2000\n",
      "107 2200\n",
      "107 2400\n",
      "107 2600\n",
      "107 2800\n",
      "107 3000\n",
      "107 3200\n",
      "107 3400\n",
      "Epoch 107: train loss: 1.0452150106430054; test loss: 1.0321390628814697\n",
      "Epoch 107: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "108 0\n",
      "108 200\n",
      "108 400\n",
      "108 600\n",
      "108 800\n",
      "108 1000\n",
      "108 1200\n",
      "108 1400\n",
      "108 1600\n",
      "108 1800\n",
      "108 2000\n",
      "108 2200\n",
      "108 2400\n",
      "108 2600\n",
      "108 2800\n",
      "108 3000\n",
      "108 3200\n",
      "108 3400\n",
      "Epoch 108: train loss: 1.0452361106872559; test loss: 1.0322219133377075\n",
      "Epoch 108: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "109 0\n",
      "109 200\n",
      "109 400\n",
      "109 600\n",
      "109 800\n",
      "109 1000\n",
      "109 1200\n",
      "109 1400\n",
      "109 1600\n",
      "109 1800\n",
      "109 2000\n",
      "109 2200\n",
      "109 2400\n",
      "109 2600\n",
      "109 2800\n",
      "109 3000\n",
      "109 3200\n",
      "109 3400\n",
      "Epoch 109: train loss: 1.0451812744140625; test loss: 1.0322219133377075\n",
      "Epoch 109: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "110 0\n",
      "110 200\n",
      "110 400\n",
      "110 600\n",
      "110 800\n",
      "110 1000\n",
      "110 1200\n",
      "110 1400\n",
      "110 1600\n",
      "110 1800\n",
      "110 2000\n",
      "110 2200\n",
      "110 2400\n",
      "110 2600\n",
      "110 2800\n",
      "110 3000\n",
      "110 3200\n",
      "110 3400\n",
      "Epoch 110: train loss: 1.0451947450637817; test loss: 1.0322271585464478\n",
      "Epoch 110: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "111 0\n",
      "111 200\n",
      "111 400\n",
      "111 600\n",
      "111 800\n",
      "111 1000\n",
      "111 1200\n",
      "111 1400\n",
      "111 1600\n",
      "111 1800\n",
      "111 2000\n",
      "111 2200\n",
      "111 2400\n",
      "111 2600\n",
      "111 2800\n",
      "111 3000\n",
      "111 3200\n",
      "111 3400\n",
      "Epoch 111: train loss: 1.0451716184616089; test loss: 1.0322285890579224\n",
      "Epoch 111: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "112 0\n",
      "112 200\n",
      "112 400\n",
      "112 600\n",
      "112 800\n",
      "112 1000\n",
      "112 1200\n",
      "112 1400\n",
      "112 1600\n",
      "112 1800\n",
      "112 2000\n",
      "112 2200\n",
      "112 2400\n",
      "112 2600\n",
      "112 2800\n",
      "112 3000\n",
      "112 3200\n",
      "112 3400\n",
      "Epoch 112: train loss: 1.0451908111572266; test loss: 1.0322283506393433\n",
      "Epoch 112: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "113 0\n",
      "113 200\n",
      "113 400\n",
      "113 600\n",
      "113 800\n",
      "113 1000\n",
      "113 1200\n",
      "113 1400\n",
      "113 1600\n",
      "113 1800\n",
      "113 2000\n",
      "113 2200\n",
      "113 2400\n",
      "113 2600\n",
      "113 2800\n",
      "113 3000\n",
      "113 3200\n",
      "113 3400\n",
      "Epoch 113: train loss: 1.0451717376708984; test loss: 1.032222867012024\n",
      "Epoch 113: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "114 0\n",
      "114 200\n",
      "114 400\n",
      "114 600\n",
      "114 800\n",
      "114 1000\n",
      "114 1200\n",
      "114 1400\n",
      "114 1600\n",
      "114 1800\n",
      "114 2000\n",
      "114 2200\n",
      "114 2400\n",
      "114 2600\n",
      "114 2800\n",
      "114 3000\n",
      "114 3200\n",
      "114 3400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 114: train loss: 1.0451772212982178; test loss: 1.0322062969207764\n",
      "Epoch 114: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "115 0\n",
      "115 200\n",
      "115 400\n",
      "115 600\n",
      "115 800\n",
      "115 1000\n",
      "115 1200\n",
      "115 1400\n",
      "115 1600\n",
      "115 1800\n",
      "115 2000\n",
      "115 2200\n",
      "115 2400\n",
      "115 2600\n",
      "115 2800\n",
      "115 3000\n",
      "115 3200\n",
      "115 3400\n",
      "Epoch 115: train loss: 1.0451548099517822; test loss: 1.0322011709213257\n",
      "Epoch 115: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "116 0\n",
      "116 200\n",
      "116 400\n",
      "116 600\n",
      "116 800\n",
      "116 1000\n",
      "116 1200\n",
      "116 1400\n",
      "116 1600\n",
      "116 1800\n",
      "116 2000\n",
      "116 2200\n",
      "116 2400\n",
      "116 2600\n",
      "116 2800\n",
      "116 3000\n",
      "116 3200\n",
      "116 3400\n",
      "Epoch 116: train loss: 1.0451633930206299; test loss: 1.0321893692016602\n",
      "Epoch 116: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "117 0\n",
      "117 200\n",
      "117 400\n",
      "117 600\n",
      "117 800\n",
      "117 1000\n",
      "117 1200\n",
      "117 1400\n",
      "117 1600\n",
      "117 1800\n",
      "117 2000\n",
      "117 2200\n",
      "117 2400\n",
      "117 2600\n",
      "117 2800\n",
      "117 3000\n",
      "117 3200\n",
      "117 3400\n",
      "Epoch 117: train loss: 1.0451308488845825; test loss: 1.0321846008300781\n",
      "Epoch 117: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "118 0\n",
      "118 200\n",
      "118 400\n",
      "118 600\n",
      "118 800\n",
      "118 1000\n",
      "118 1200\n",
      "118 1400\n",
      "118 1600\n",
      "118 1800\n",
      "118 2000\n",
      "118 2200\n",
      "118 2400\n",
      "118 2600\n",
      "118 2800\n",
      "118 3000\n",
      "118 3200\n",
      "118 3400\n",
      "Epoch 118: train loss: 1.0451573133468628; test loss: 1.0321910381317139\n",
      "Epoch 118: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "119 0\n",
      "119 200\n",
      "119 400\n",
      "119 600\n",
      "119 800\n",
      "119 1000\n",
      "119 1200\n",
      "119 1400\n",
      "119 1600\n",
      "119 1800\n",
      "119 2000\n",
      "119 2200\n",
      "119 2400\n",
      "119 2600\n",
      "119 2800\n",
      "119 3000\n",
      "119 3200\n",
      "119 3400\n",
      "Epoch 119: train loss: 1.045117735862732; test loss: 1.0322074890136719\n",
      "Epoch 119: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "120 0\n",
      "120 200\n",
      "120 400\n",
      "120 600\n",
      "120 800\n",
      "120 1000\n",
      "120 1200\n",
      "120 1400\n",
      "120 1600\n",
      "120 1800\n",
      "120 2000\n",
      "120 2200\n",
      "120 2400\n",
      "120 2600\n",
      "120 2800\n",
      "120 3000\n",
      "120 3200\n",
      "120 3400\n",
      "Epoch 120: train loss: 1.0451204776763916; test loss: 1.0321409702301025\n",
      "Epoch 120: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "121 0\n",
      "121 200\n",
      "121 400\n",
      "121 600\n",
      "121 800\n",
      "121 1000\n",
      "121 1200\n",
      "121 1400\n",
      "121 1600\n",
      "121 1800\n",
      "121 2000\n",
      "121 2200\n",
      "121 2400\n",
      "121 2600\n",
      "121 2800\n",
      "121 3000\n",
      "121 3200\n",
      "121 3400\n",
      "Epoch 121: train loss: 1.0450493097305298; test loss: 1.0320707559585571\n",
      "Epoch 121: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "122 0\n",
      "122 200\n",
      "122 400\n",
      "122 600\n",
      "122 800\n",
      "122 1000\n",
      "122 1200\n",
      "122 1400\n",
      "122 1600\n",
      "122 1800\n",
      "122 2000\n",
      "122 2200\n",
      "122 2400\n",
      "122 2600\n",
      "122 2800\n",
      "122 3000\n",
      "122 3200\n",
      "122 3400\n",
      "Epoch 122: train loss: 1.0451202392578125; test loss: 1.032151222229004\n",
      "Epoch 122: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "123 0\n",
      "123 200\n",
      "123 400\n",
      "123 600\n",
      "123 800\n",
      "123 1000\n",
      "123 1200\n",
      "123 1400\n",
      "123 1600\n",
      "123 1800\n",
      "123 2000\n",
      "123 2200\n",
      "123 2400\n",
      "123 2600\n",
      "123 2800\n",
      "123 3000\n",
      "123 3200\n",
      "123 3400\n",
      "Epoch 123: train loss: 1.0450283288955688; test loss: 1.0319126844406128\n",
      "Epoch 123: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "124 0\n",
      "124 200\n",
      "124 400\n",
      "124 600\n",
      "124 800\n",
      "124 1000\n",
      "124 1200\n",
      "124 1400\n",
      "124 1600\n",
      "124 1800\n",
      "124 2000\n",
      "124 2200\n",
      "124 2400\n",
      "124 2600\n",
      "124 2800\n",
      "124 3000\n",
      "124 3200\n",
      "124 3400\n",
      "Epoch 124: train loss: 1.0451186895370483; test loss: 1.03224778175354\n",
      "Epoch 124: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "125 0\n",
      "125 200\n",
      "125 400\n",
      "125 600\n",
      "125 800\n",
      "125 1000\n",
      "125 1200\n",
      "125 1400\n",
      "125 1600\n",
      "125 1800\n",
      "125 2000\n",
      "125 2200\n",
      "125 2400\n",
      "125 2600\n",
      "125 2800\n",
      "125 3000\n",
      "125 3200\n",
      "125 3400\n",
      "Epoch 125: train loss: 1.0451346635818481; test loss: 1.032178282737732\n",
      "Epoch 125: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "126 0\n",
      "126 200\n",
      "126 400\n",
      "126 600\n",
      "126 800\n",
      "126 1000\n",
      "126 1200\n",
      "126 1400\n",
      "126 1600\n",
      "126 1800\n",
      "126 2000\n",
      "126 2200\n",
      "126 2400\n",
      "126 2600\n",
      "126 2800\n",
      "126 3000\n",
      "126 3200\n",
      "126 3400\n",
      "Epoch 126: train loss: 1.0451202392578125; test loss: 1.0320720672607422\n",
      "Epoch 126: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "127 0\n",
      "127 200\n",
      "127 400\n",
      "127 600\n",
      "127 800\n",
      "127 1000\n",
      "127 1200\n",
      "127 1400\n",
      "127 1600\n",
      "127 1800\n",
      "127 2000\n",
      "127 2200\n",
      "127 2400\n",
      "127 2600\n",
      "127 2800\n",
      "127 3000\n",
      "127 3200\n",
      "127 3400\n",
      "Epoch 127: train loss: 1.0450223684310913; test loss: 1.0319572687149048\n",
      "Epoch 127: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "128 0\n",
      "128 200\n",
      "128 400\n",
      "128 600\n",
      "128 800\n",
      "128 1000\n",
      "128 1200\n",
      "128 1400\n",
      "128 1600\n",
      "128 1800\n",
      "128 2000\n",
      "128 2200\n",
      "128 2400\n",
      "128 2600\n",
      "128 2800\n",
      "128 3000\n",
      "128 3200\n",
      "128 3400\n",
      "Epoch 128: train loss: 1.0450944900512695; test loss: 1.0321719646453857\n",
      "Epoch 128: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "129 0\n",
      "129 200\n",
      "129 400\n",
      "129 600\n",
      "129 800\n",
      "129 1000\n",
      "129 1200\n",
      "129 1400\n",
      "129 1600\n",
      "129 1800\n",
      "129 2000\n",
      "129 2200\n",
      "129 2400\n",
      "129 2600\n",
      "129 2800\n",
      "129 3000\n",
      "129 3200\n",
      "129 3400\n",
      "Epoch 129: train loss: 1.045084834098816; test loss: 1.032090187072754\n",
      "Epoch 129: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "130 0\n",
      "130 200\n",
      "130 400\n",
      "130 600\n",
      "130 800\n",
      "130 1000\n",
      "130 1200\n",
      "130 1400\n",
      "130 1600\n",
      "130 1800\n",
      "130 2000\n",
      "130 2200\n",
      "130 2400\n",
      "130 2600\n",
      "130 2800\n",
      "130 3000\n",
      "130 3200\n",
      "130 3400\n",
      "Epoch 130: train loss: 1.0450935363769531; test loss: 1.032126545906067\n",
      "Epoch 130: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "131 0\n",
      "131 200\n",
      "131 400\n",
      "131 600\n",
      "131 800\n",
      "131 1000\n",
      "131 1200\n",
      "131 1400\n",
      "131 1600\n",
      "131 1800\n",
      "131 2000\n",
      "131 2200\n",
      "131 2400\n",
      "131 2600\n",
      "131 2800\n",
      "131 3000\n",
      "131 3200\n",
      "131 3400\n",
      "Epoch 131: train loss: 1.0450735092163086; test loss: 1.0321122407913208\n",
      "Epoch 131: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "132 0\n",
      "132 200\n",
      "132 400\n",
      "132 600\n",
      "132 800\n",
      "132 1000\n",
      "132 1200\n",
      "132 1400\n",
      "132 1600\n",
      "132 1800\n",
      "132 2000\n",
      "132 2200\n",
      "132 2400\n",
      "132 2600\n",
      "132 2800\n",
      "132 3000\n",
      "132 3200\n",
      "132 3400\n",
      "Epoch 132: train loss: 1.0450671911239624; test loss: 1.0321444272994995\n",
      "Epoch 132: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "133 0\n",
      "133 200\n",
      "133 400\n",
      "133 600\n",
      "133 800\n",
      "133 1000\n",
      "133 1200\n",
      "133 1400\n",
      "133 1600\n",
      "133 1800\n",
      "133 2000\n",
      "133 2200\n",
      "133 2400\n",
      "133 2600\n",
      "133 2800\n",
      "133 3000\n",
      "133 3200\n",
      "133 3400\n",
      "Epoch 133: train loss: 1.0450513362884521; test loss: 1.0321767330169678\n",
      "Epoch 133: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "134 0\n",
      "134 200\n",
      "134 400\n",
      "134 600\n",
      "134 800\n",
      "134 1000\n",
      "134 1200\n",
      "134 1400\n",
      "134 1600\n",
      "134 1800\n",
      "134 2000\n",
      "134 2200\n",
      "134 2400\n",
      "134 2600\n",
      "134 2800\n",
      "134 3000\n",
      "134 3200\n",
      "134 3400\n",
      "Epoch 134: train loss: 1.0450031757354736; test loss: 1.032135009765625\n",
      "Epoch 134: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "135 0\n",
      "135 200\n",
      "135 400\n",
      "135 600\n",
      "135 800\n",
      "135 1000\n",
      "135 1200\n",
      "135 1400\n",
      "135 1600\n",
      "135 1800\n",
      "135 2000\n",
      "135 2200\n",
      "135 2400\n",
      "135 2600\n",
      "135 2800\n",
      "135 3000\n",
      "135 3200\n",
      "135 3400\n",
      "Epoch 135: train loss: 1.0450032949447632; test loss: 1.0320472717285156\n",
      "Epoch 135: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "136 0\n",
      "136 200\n",
      "136 400\n",
      "136 600\n",
      "136 800\n",
      "136 1000\n",
      "136 1200\n",
      "136 1400\n",
      "136 1600\n",
      "136 1800\n",
      "136 2000\n",
      "136 2200\n",
      "136 2400\n",
      "136 2600\n",
      "136 2800\n",
      "136 3000\n",
      "136 3200\n",
      "136 3400\n",
      "Epoch 136: train loss: 1.0449087619781494; test loss: 1.0319381952285767\n",
      "Epoch 136: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "137 0\n",
      "137 200\n",
      "137 400\n",
      "137 600\n",
      "137 800\n",
      "137 1000\n",
      "137 1200\n",
      "137 1400\n",
      "137 1600\n",
      "137 1800\n",
      "137 2000\n",
      "137 2200\n",
      "137 2400\n",
      "137 2600\n",
      "137 2800\n",
      "137 3000\n",
      "137 3200\n",
      "137 3400\n",
      "Epoch 137: train loss: 1.0450325012207031; test loss: 1.032136082649231\n",
      "Epoch 137: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "138 0\n",
      "138 200\n",
      "138 400\n",
      "138 600\n",
      "138 800\n",
      "138 1000\n",
      "138 1200\n",
      "138 1400\n",
      "138 1600\n",
      "138 1800\n",
      "138 2000\n",
      "138 2200\n",
      "138 2400\n",
      "138 2600\n",
      "138 2800\n",
      "138 3000\n",
      "138 3200\n",
      "138 3400\n",
      "Epoch 138: train loss: 1.0450093746185303; test loss: 1.0320478677749634\n",
      "Epoch 138: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "139 0\n",
      "139 200\n",
      "139 400\n",
      "139 600\n",
      "139 800\n",
      "139 1000\n",
      "139 1200\n",
      "139 1400\n",
      "139 1600\n",
      "139 1800\n",
      "139 2000\n",
      "139 2200\n",
      "139 2400\n",
      "139 2600\n",
      "139 2800\n",
      "139 3000\n",
      "139 3200\n",
      "139 3400\n",
      "Epoch 139: train loss: 1.0450191497802734; test loss: 1.032114028930664\n",
      "Epoch 139: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "140 0\n",
      "140 200\n",
      "140 400\n",
      "140 600\n",
      "140 800\n",
      "140 1000\n",
      "140 1200\n",
      "140 1400\n",
      "140 1600\n",
      "140 1800\n",
      "140 2000\n",
      "140 2200\n",
      "140 2400\n",
      "140 2600\n",
      "140 2800\n",
      "140 3000\n",
      "140 3200\n",
      "140 3400\n",
      "Epoch 140: train loss: 1.0449992418289185; test loss: 1.0321468114852905\n",
      "Epoch 140: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "141 0\n",
      "141 200\n",
      "141 400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141 600\n",
      "141 800\n",
      "141 1000\n",
      "141 1200\n",
      "141 1400\n",
      "141 1600\n",
      "141 1800\n",
      "141 2000\n",
      "141 2200\n",
      "141 2400\n",
      "141 2600\n",
      "141 2800\n",
      "141 3000\n",
      "141 3200\n",
      "141 3400\n",
      "Epoch 141: train loss: 1.0449579954147339; test loss: 1.0321319103240967\n",
      "Epoch 141: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "142 0\n",
      "142 200\n",
      "142 400\n",
      "142 600\n",
      "142 800\n",
      "142 1000\n",
      "142 1200\n",
      "142 1400\n",
      "142 1600\n",
      "142 1800\n",
      "142 2000\n",
      "142 2200\n",
      "142 2400\n",
      "142 2600\n",
      "142 2800\n",
      "142 3000\n",
      "142 3200\n",
      "142 3400\n",
      "Epoch 142: train loss: 1.0449190139770508; test loss: 1.0320616960525513\n",
      "Epoch 142: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "143 0\n",
      "143 200\n",
      "143 400\n",
      "143 600\n",
      "143 800\n",
      "143 1000\n",
      "143 1200\n",
      "143 1400\n",
      "143 1600\n",
      "143 1800\n",
      "143 2000\n",
      "143 2200\n",
      "143 2400\n",
      "143 2600\n",
      "143 2800\n",
      "143 3000\n",
      "143 3200\n",
      "143 3400\n",
      "Epoch 143: train loss: 1.0449802875518799; test loss: 1.0321311950683594\n",
      "Epoch 143: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "144 0\n",
      "144 200\n",
      "144 400\n",
      "144 600\n",
      "144 800\n",
      "144 1000\n",
      "144 1200\n",
      "144 1400\n",
      "144 1600\n",
      "144 1800\n",
      "144 2000\n",
      "144 2200\n",
      "144 2400\n",
      "144 2600\n",
      "144 2800\n",
      "144 3000\n",
      "144 3200\n",
      "144 3400\n",
      "Epoch 144: train loss: 1.0448678731918335; test loss: 1.031990647315979\n",
      "Epoch 144: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "145 0\n",
      "145 200\n",
      "145 400\n",
      "145 600\n",
      "145 800\n",
      "145 1000\n",
      "145 1200\n",
      "145 1400\n",
      "145 1600\n",
      "145 1800\n",
      "145 2000\n",
      "145 2200\n",
      "145 2400\n",
      "145 2600\n",
      "145 2800\n",
      "145 3000\n",
      "145 3200\n",
      "145 3400\n",
      "Epoch 145: train loss: 1.0449625253677368; test loss: 1.0321879386901855\n",
      "Epoch 145: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "146 0\n",
      "146 200\n",
      "146 400\n",
      "146 600\n",
      "146 800\n",
      "146 1000\n",
      "146 1200\n",
      "146 1400\n",
      "146 1600\n",
      "146 1800\n",
      "146 2000\n",
      "146 2200\n",
      "146 2400\n",
      "146 2600\n",
      "146 2800\n",
      "146 3000\n",
      "146 3200\n",
      "146 3400\n",
      "Epoch 146: train loss: 1.044924259185791; test loss: 1.032069444656372\n",
      "Epoch 146: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "147 0\n",
      "147 200\n",
      "147 400\n",
      "147 600\n",
      "147 800\n",
      "147 1000\n",
      "147 1200\n",
      "147 1400\n",
      "147 1600\n",
      "147 1800\n",
      "147 2000\n",
      "147 2200\n",
      "147 2400\n",
      "147 2600\n",
      "147 2800\n",
      "147 3000\n",
      "147 3200\n",
      "147 3400\n",
      "Epoch 147: train loss: 1.0449265241622925; test loss: 1.032019853591919\n",
      "Epoch 147: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "148 0\n",
      "148 200\n",
      "148 400\n",
      "148 600\n",
      "148 800\n",
      "148 1000\n",
      "148 1200\n",
      "148 1400\n",
      "148 1600\n",
      "148 1800\n",
      "148 2000\n",
      "148 2200\n",
      "148 2400\n",
      "148 2600\n",
      "148 2800\n",
      "148 3000\n",
      "148 3200\n",
      "148 3400\n",
      "Epoch 148: train loss: 1.0448381900787354; test loss: 1.0319260358810425\n",
      "Epoch 148: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n",
      "149 0\n",
      "149 200\n",
      "149 400\n",
      "149 600\n",
      "149 800\n",
      "149 1000\n",
      "149 1200\n",
      "149 1400\n",
      "149 1600\n",
      "149 1800\n",
      "149 2000\n",
      "149 2200\n",
      "149 2400\n",
      "149 2600\n",
      "149 2800\n",
      "149 3000\n",
      "149 3200\n",
      "149 3400\n",
      "Epoch 149: train loss: 1.0449793338775635; test loss: 1.0321043729782104\n",
      "Epoch 149: train accuracy: 0.7395950506186727; test accuracy: 0.7109111361079865\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "train_losses = []\n",
    "test_losses = []\n",
    "train_accuracies = []\n",
    "test_accuracies = []\n",
    "for n_epoch in range(n_epoches):\n",
    "    # create permutation for batch training\n",
    "    # To-Do: add permutation for SGD...But it is slow.\n",
    "#    permutation = torch.randperm(x_train_images_norm_torch.size()[0])\n",
    "    for i in range(0, x_train_images_norm_torch.size()[0], batch_size):\n",
    "        print(n_epoch, i)\n",
    "        # clear gradients first (for each iteration!)!\n",
    "        optim.zero_grad()\n",
    "        # forward pass\n",
    "        batch_x, batch_y = x_train_images_norm_torch[i:i+batch_size, :, :, :].to(device), y_train_torch[i:i+batch_size].to(device)\n",
    "        \n",
    "        batch_x_100 = x_train_images_norm_torch_100[i:i+batch_size, :, :, :].to(device)\n",
    "        batch_x_50 = x_train_images_norm_torch_50[i:i+batch_size, :, :, :].to(device)\n",
    "        \n",
    "        batch_y_pred_train = cnn_net(batch_x,batch_x_100,batch_x_50)\n",
    "        # loss \n",
    "        loss = criterion(batch_y_pred_train.squeeze(), batch_y)\n",
    "        # compute gradients\n",
    "        loss.backward()\n",
    "        # one step optim\n",
    "        optim.step()\n",
    "\n",
    "    # eval training accuracy\n",
    "    with torch.no_grad():\n",
    "        y_pred_train = cnn_net(x_train_images_norm_torch.to(device),x_train_images_norm_torch_100.to(device),x_train_images_norm_torch_50.to(device))\n",
    "        loss_train = criterion(y_pred_train.squeeze(), y_train_torch.to(device))\n",
    "        train_losses.append(loss_train)\n",
    "        \n",
    "        _, predict_train = torch.max(y_pred_train, axis = 1)\n",
    "        accuracy_train = (predict_train == y_train_torch.to(device)).sum().item()/n_train  \n",
    "        train_accuracies.append(accuracy_train)\n",
    "        # evaluate testing sets step-wise\n",
    "        cnn_net.eval()\n",
    "        y_pred_test = cnn_net(x_test_images_norm_torch.to(device),x_test_images_norm_torch_100.to(device),x_test_images_norm_torch_50.to(device))\n",
    "       \n",
    "        loss_test = criterion(y_pred_test.squeeze().to(device), y_test_torch.to(device))\n",
    "        test_losses.append(loss_test)\n",
    "        _, predict_test = torch.max(y_pred_test.to(device), axis = 1)\n",
    "        accuracy_test = (predict_test == y_test_torch.to(device)).sum().item()/n_test\n",
    "        test_accuracies.append(accuracy_test)\n",
    "        # print info\n",
    "        if n_epoch % 1 == 0:\n",
    "            print('Epoch {}: train loss: {}; test loss: {}'.format(n_epoch, loss.item(), loss_test.item()))\n",
    "            print('Epoch {}: train accuracy: {}; test accuracy: {}'.format(n_epoch, accuracy_train, accuracy_test))\n",
    "\n",
    "\n",
    "# notes:\n",
    "# CPU training: about 30 mins, with SIMPLEST CNN architecture, 20 epoches and 200 batch_size. \n",
    "# training accuracy: 60%; testing accuracy: 60%.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f168029b2e8>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcU9Xd+PHPN5lkMvvOOsCwySLLgAPiVkGtArYuj31at2pbrd1sq22t2vbR+tjnp1ZbLa1VqUWttrS2VksrKi4sLogMCIqyL8KwOAszw+xbzu+Pk2EyQ2YBMnND8n2/XnklOefm5psL883JueeeI8YYlFJKxQ6X0wEopZTqW5r4lVIqxmjiV0qpGKOJXymlYowmfqWUijGa+JVSKsZo4ldKqRijiV8ppWKMJn6llIoxcU4HEEp2drbJy8tzOgyllDphrFmzptQYk9OTbXuU+EVkAfA5oNgYMyFE/cXA3YAfaAZuMsa8Fai7FvhZYNNfGGOe6u798vLyKCws7EloSimlABH5pKfb9rSr50lgdhf1rwOTjTH5wNeAxwOBZAJ3AqcC04E7RSSjp8EppZQKvx4lfmPMCuBgF/XVpm22tySg9fEFwKvGmIPGmHLgVbr+AlFKKdXLwnZyV0QuFZFNwIvYVj/AYGBP0GZFgTKllFIOCdvJXWPM88DzIvIZbH//eYCE2jTU60XkBuAGgKFDh4YrLKVUhGlqaqKoqIj6+nqnQzkh+Xw+cnNz8Xg8x7yPsI/qMcasEJGRIpKNbeHPDKrOBZZ18rr5wHyAgoICXSRAqShVVFRESkoKeXl5iIRqG6rOGGMoKyujqKiI4cOHH/N+wtLVIyKjJPAvKCJTAS9QBrwCnC8iGYGTuucHypRSMaq+vp6srCxN+sdARMjKyjruX0s9Hc65ENtyzxaRIuxIHQ+AMeZR4DLgGhFpAuqALwVO9h4UkbuB1YFd/a8xptOTxEqp2KBJ/9iF49j1KPEbY67opv4+4L5O6hYAC44+tGO3bHMxeVlJ5GUn9eXbKqXUCSHqpmyob2rh0af/wl9fW+l0KEqpCFNRUcHvf//7Y3rt3Llzqaio6PH2P//5z3nggQeO6b16W9Ql/vd3lfG4+x5O29ftBcJKqRjTVeJvaWnp8rWLFy8mPT29N8Lqc1GX+LduXEuy1JPQUOp0KEqpCHPbbbexfft28vPzueWWW1i2bBmzZs3iyiuvZOLEiQBccsklnHLKKZx88snMnz//8Gvz8vIoLS1l165djBs3jq9//eucfPLJnH/++dTV1XX5vuvWrWPGjBlMmjSJSy+9lPLycgDmzZvH+PHjmTRpEpdffjkAy5cvJz8/n/z8fKZMmUJVVVXYj0NETtJ2PKp2rAEgobnS4UiUUt25698f8fG+Q2Hd5/hBqdz5+ZND1t17771s2LCBdevWAbBs2TLee+89NmzYcHh45IIFC8jMzKSuro5p06Zx2WWXkZWV1W4/W7duZeHChfzhD3/gi1/8Is899xxXX311pzFdc801/Pa3v+Xss8/mjjvu4K677uKhhx7i3nvvZefOncTHxx/uRnrggQd4+OGHOeOMM6iursbn84XjsLQTVS3+huYWkg5+CEByiyZ+pVT3pk+f3m5M/Lx585g8eTIzZsxgz549bN269YjXDB8+nPz8fABOOeUUdu3a1en+Kysrqaio4Oyzzwbg2muvZcWKFQBMmjSJq666imeeeYa4ONsOP+OMM/jBD37AvHnzqKioOFweTlHV4v+gqJJx7AQg1YS3FaGUCr/OWuZ9KSmpbfTfsmXLeO2111i5ciWJiYnMnDkz5Jj5+Pj4w4/dbne3XT2defHFF1mxYgWLFi3i7rvv5qOPPuK2227jwgsvZPHixcyYMYPXXnuNsWPHHtP+OxNVLf5V20sYH5iZNJ0q6hqaHI5IKRVJUlJSuuwzr6ysJCMjg8TERDZt2sS777573O+ZlpZGRkYGb775JgBPP/00Z599Nn6/nz179jBr1ix++ctfUlFRQXV1Ndu3b2fixInceuutFBQUsGnTpuOOoaOoavHv2voRKVJHecpJZFRtoaK8hIQBg5wOSykVIbKysjjjjDOYMGECc+bM4cILL2xXP3v2bB599FEmTZrEmDFjmDFjRlje96mnnuKb3/wmtbW1jBgxgieeeIKWlhauvvpqKisrMcZw8803k56ezv/8z/+wdOlS3G4348ePZ86cOWGJIZi0zaYcOQoKCszRLsTS1OLntrvu5Feueew86WsM37KAbV9czqjx+b0UpVLqWGzcuJFx48Y5HcYJLdQxFJE1xpiCnrw+arp6XCL8cGI9fpeX+sGnAVBX+anDUSmlVOSJmq4et0sYVLsZBpyML9N27zQc0rH8SinVUdS0+DEG9q+HgfkkpfcHoKmqxOGglFIq8kRNix9/M5x7B+SMJTXTJn5TU+ZwUEopFXmiJ/G7PTDtOgB8xlBvPEidJn6llOooerp6golQKam468udjkQppSJOdCZ+oMqdhrdRE79Sqs3xTMsM8NBDD1FbWxuybubMmRztMHSnRG3ir3OnkdDU87mzlVLRrzcT/4kkahN/vTedJJ2oTSkVpOO0zAD3338/06ZNY9KkSdx5550A1NTUcOGFFzJ58mQmTJjA3/72N+bNm8e+ffuYNWsWs2bN6vJ9Fi5cyMSJE5kwYQK33norYOf7/8pXvsKECROYOHEiDz74IBB6aubeFj0ndztois8kpVonalMqor10Gxz4MLz7HDAR5twbsqrjtMxLlixh69atvPfeexhjuOiii1ixYgUlJSUMGjSIF198EbBz+KSlpfHrX/+apUuXkp2d3enb79u3j1tvvZU1a9aQkZHB+eefzwsvvMCQIUPYu3cvGzZsADg8DXOoqZl7W9S2+P0JmaRRjWnRidqUUqEtWbKEJUuWMGXKFKZOncqmTZvYunUrEydO5LXXXuPWW2/lzTffJC0trcf7XL16NTNnziQnJ4e4uDiuuuoqVqxYwYgRI9ixYwff/e53efnll0lNTQVCT83c26K2xU9iJgA1FSUkZ+lEbUpFpE5a5n3FGMPtt9/ON77xjSPq1qxZw+LFi7n99ts5//zzueOOO3q8z1AyMjJYv349r7zyCg8//DDPPvssCxYsCDk1c29/AURti9+dbH+KVR3U+XqUUlbHaZkvuOACFixYQHV1NQB79+6luLiYffv2kZiYyNVXX82PfvQj1q5dG/L1oZx66qksX76c0tJSWlpaWLhwIWeffTalpaX4/X4uu+wy7r77btauXdvp1My9LWpb/J7kHABqK3TaBqWU1XFa5vvvv5+NGzdy2ml2Ysfk5GSeeeYZtm3bxi233ILL5cLj8fDII48AcMMNNzBnzhwGDhzI0qVLQ77HwIEDueeee5g1axbGGObOncvFF1/M+vXr+epXv4rf7wfgnnvu6XRq5t4WNdMyd7Rh7VtMWHQhH5/1MOPP7XwtTKVU39JpmY+fTsvcicS0fgA0VekMnUopFazbxC8iC0SkWEQ2dFJ/lYh8ELi9IyKTg+p2iciHIrJORPr0kraUwERtLTWa+JVSKlhPWvxPArO7qN8JnG2MmQTcDczvUD/LGJPf058g4ZKWkkKNiSd33xKYPwuevQZKt/VlCEqpTkRiF/OJIhzHrtvEb4xZARzsov4dY0zrpDjvArnHHVUYeONc7KUf2TVbKKttxr/1Nfj9qbDmSadDUyqm+Xw+ysrKNPkfA2MMZWVl+Hy+49pPuEf1XAe8FPTcAEtExACPGWM6/hroVXek/R/bimsoPZDGhLQG/pX+AO41T8IpX+nLMJRSQXJzcykqKqKkREfcHQufz0du7vG1r8OW+EVkFjbxnxlUfIYxZp+I9ANeFZFNgV8QoV5/A3ADwNChQ8MS06PfnENxVQNF5bVc/1QhKxNHc+ahpXa1LpGwvIdS6uh4PB6GDx/udBgxLSyjekRkEvA4cLEx5vDqJ8aYfYH7YuB5YHpn+zDGzDfGFBhjCnJycsIRFumJXk7qn8I5Y/tz46xRvFGcAg2VUNtpz5VSSkW94078IjIU+CfwZWPMlqDyJBFJaX0MnA+EHBnUF7577mgaUofZJwe3OxWGUko5rtuuHhFZCMwEskWkCLgT8AAYYx4F7gCygN+L7T5pDozg6Q88HyiLA/5ijHm5Fz5Dj3jcLpIHjoEdwMEdMKTTHx9KKRXVuk38xpgruqm/Hrg+RPkOYPKRr3BO0oCRtGwXTMnW6J2rQimluhG1V+6Gkpudxl6TTd2nW50ORSmlHBNTiX9YViK7zABM2Q6nQ1FKKcfEVOIfkmkTv+/QLjukUymlYlBMdXXnJMez1zUQb3OVHdJ5aC801+uJXqVUTImpFr+IUJucZ58UfwwLr4AXf+BoTEop1ddiqsUPYDJHQC3w+l1wqAhMi9MhKaVUn4qpFj9AQr8RtBiBotW2oPag9vcrpWJKzCX+1iGdAIw+H1oaoLHG2aCUUqoPxVziH5qVyDv+kykfch6Mu8gW1pZ1/SKllIoisZf4MxO5rfkGluY/xK66eFtYp5O2KaViR8wl/tyMBERg0Qf7uf3lfbZQW/xKqRgSc4k/Ps7NwFQfyzaXUNycaAtry7t+kVJKRZGYG84JMHZgKiLC9MGjYTva4ldKxZSYTPy/vWIKBnjmnR34twkt1aV2nmmllIoBMZn4k+Ltx05P8lFJEt5DJZr4lVIxI+b6+IOlJ3o5aFJoqtauHqVU7IjpxJ+R6KGCZO3jV0rFlNhO/Em2xS86jl8pFUNiOvGnJ3qoMMnENehwTqVU7IjtxJ/g5SApxDdWOB2KUkr1mZhO/N44FzXuNOL8DdBY63Q4SinVJ2I68QM0etPtAz3Bq5SKETGf+Ft8mfaBnuBVSsWImE/8/oQM+0Bb/EqpGBHzid+VGFiUpVZb/Eqp2BDzid+TnGUfaOJXSsWIbhO/iCwQkWIR2dBJ/VUi8kHg9o6ITA6qmy0im0Vkm4jcFs7Aw8WbahO/v6bU4UiUUqpv9KTF/yQwu4v6ncDZxphJwN3AfAARcQMPA3OA8cAVIjL+uKLtBelJCVSYJBqqNPErpWJDt4nfGLMC6LQfxBjzjjGm9dLXd4HcwOPpwDZjzA5jTCPwV+Di44w37DKSvJSbZJo18SulYkS4+/ivA14KPB4M7AmqKwqUhSQiN4hIoYgUlpSUhDmszqUneiknBb+O6lFKxYiwJX4RmYVN/Le2FoXYzHT2emPMfGNMgTGmICcnJ1xhdSsz0Uu5SUE08SulYkRYEr+ITAIeBy42xrRm0CJgSNBmucC+cLxfOKUnethshpBcuRVqNPkrpaLfcSd+ERkK/BP4sjFmS1DVamC0iAwXES9wObDoeN8v3DKSvPynZQYu0wwbIy48pZQKu54M51wIrATGiEiRiFwnIt8UkW8GNrkDyAJ+LyLrRKQQwBjTDNwIvAJsBJ41xnzUK5/iOCR53Wx15VHmGwYbnnM6HKWU6nXdrrlrjLmim/rrges7qVsMLD620PqGiJCeGM/7qedw3q4n4dB+SB3odFhKKdVrYv7KXbBLMK6I/wxg4OMXnA5HKaV6lSZ+7JDOTc0DYcBE2PBPp8NRSqlepYkfO6SzrLoBhp8N+9eD3+90SEop1Ws08QOj+iWzq6yWprTh0NIAVfudDkkppXqNJn5gUm4aLX7DTn8/W3Bwh7MBKaVUL9LED0zKtcsvrq8NrMZVvtPBaJRSqndp4gcGpPnolxLPqlIfuOLgoCZ+pVT00sQfMCk3nff3VkP6UO3qUUpFNU38AZNy09hRWkNz+nDt6lFKRTVN/AGTctMwBko8g2xXj+l0IlGllDqhaeIPaD3Bu7OlHzQcsmvwlm6D9X9zODKllAovTfwBmUlecjMSWF8TNLJnyU/hhW/pBV1KqaiiiT/IpNw0Vlak2id7VsHWV8G0QGOVs4EppVQYaeIPMiwriTWHUjEIvD3PJn2AugpnA1NKqTDSxB9kUJqPmhYP/uSBUH0AJHB46jXxK6Wihyb+IAPTEgCoTRlqC8ZdZO+1xa+UiiKa+IMMTPcBUO4bCm4vTLvOVtRXOhiVUkqFlyb+IIMCLf63B18H1yyCjOG2Qrt6lFJRpNulF2NJeqKHBI+bbfWpMGw8NARG82hXj1IqimiLP4iIMDDdx/7KOlvgTQZxa4tfKRVVNPF3MCgtgX0V9faJCPjStMWvlIoqmvg7GJgW1OIHSEjXFr9SKqpo4u9gYHoCxVUNNLUEpmnwpWuLXykVVTTxdzAozYcx8OmhQHePtviVUlFGE38HA9PtkM79lYHEry1+pVSU6Tbxi8gCESkWkQ2d1I8VkZUi0iAiP+pQt0tEPhSRdSJSGK6ge9OgNHsR176KOowx+H3pegGXUiqq9KTF/yQwu4v6g8D3gAc6qZ9ljMk3xhQcZWyOCG7xz3t9G3/5oNJ29ejCLEqpKNFt4jfGrMAm987qi40xq4GmcAbmlOT4OFJ8cazfU8Ejy7exu9YL/mZorHE6NKWUCove7uM3wBIRWSMiN/Tye4XNoLQEXtpwgPomP5Uk20I9wauUihK9nfjPMMZMBeYA3xGRz3S2oYjcICKFIlJYUlLSy2F1rXWytsHpCRwyibZQT/AqpaJEryZ+Y8y+wH0x8DwwvYtt5xtjCowxBTk5Ob0ZVreGZibicQu3zhlLJUm2UFv8Sqko0WuJX0SSRCSl9TFwPhByZFCkufGcUfzjm6czcXAalSaQ+Osq7Nq7tZ2e7lBKqRNCt7NzishCYCaQLSJFwJ2AB8AY86iIDAAKgVTALyI3AeOBbOB5EWl9n78YY17ujQ8Rbv1SfPRL8VFZ19S+xb/2SXjlZ3DTB5CU7WiMSil1rLpN/MaYK7qpPwDkhqg6BEw+xrgiQqovjhoJnNytq4Ci1dBUA5sXw9RrnA1OKaWOkV652wURwZOYhh+xF3HtXWsrNv7H2cCUUuo4aOLvRmayjzpXMpRthcrdEJ8GO5a2LdKilFInGE383chM8lIlSbB9qS048/vQ0ghbX3U2MKWUOkaa+LuRmeS1I3vqK+xqXNNvgMRs2KTdPUqpE5Mm/m5kJXkp9wcu4uo/HuJTYOxc2LIEmhucDU4ppY6BJv5uZCR5KWuxE7c1DpjC29tKYcQsaKyCkk0OR6eUUkdPE383slq7eoC3avO4+o+rqEgeaSuLNfErpU48mvi7kZkUz6HARG1v1w/DGNja3B9cHijZ6HB0Sil19DTxdyMzycu7/rGUDzyL5QczANhR3ghZo6BYE79S6sSjib8bWclelvmn8Nopj7CjzC7HuKOkBvqN1cSvlDohaeLvRmaSF4BVOw/iDyzCtaO0BnLGQcUnukCLUuqEo4m/G+kJHgBWbi8DYEz/FHaUVNsWP0DJZqdCU0qpY6KJvxtxbhfpiR72VtTh87iYOSaH3Qdrac5qTfw6skcpdWLRxN8Drd09Y/qnMLJfMk0thiIZAG6v9vMrpU44mvh7ICuQ+McOSGVEth3Tv/NgA2Sf1HmLf+8aMKavQlRKqR7TxN8Dh1v8A1IYkWPH9G8vqYacTkb2fPIO/OEc2P56X4aplFI9oom/BzKT4gEYOzCFjEQPaQkedpYGhnRW7oHq4vYv2LPK3u9e1ceRKqVU9zTx90B2cltXj4gwIifJjuUfdZ7t518wG0q3tb1g3/uB+7UORKuUUl3TxN8Dl08fyq/+e/LhLp/h2Um2xT9oClz7b7s61+PnQtUB+4LWxK/9/EqpCKSJvwcGpydw2SltywqPyE7iwKF6qhuaYegMuOYFO1//x4ugpgwqdkPmSKgrh/JdzgWulFIhaOI/BicPTgNg/Z4KWzBgImSNhs0vwv5Aa7/ga/Zeu3uUUhFGE/8xKBiWgUtg1Y6ytsIxc2DXW7BjuX2efyW449sWaN+3Dlqa+z5YpZTqQBP/MUjxeTh5UBqrdh5sKxx7IfiboXCBbf0nZsLASTbxr1sI88+GD591LmillArQxH+MTh2eyft7KqhvarEFudPsWryN1fakL8CgqfZE74s/tM93rnAmWKWUCqKJ/xidOiKLxmZ/Wz+/yw0nzbaPWxP/4FOguQ7i4mHoafDJ284EG2nWLYS1f3I6CqViVreJX0QWiEixiGzopH6siKwUkQYR+VGHutkisllEtonIbeEKOhJMz8tEhPbdPSdfau+HzrD3w8+C1Fy49FEYf4kd7VOxp++DjTRrnoTCJ5yOQqmY1ZMW/5PA7C7qDwLfAx4ILhQRN/AwMAcYD1whIuOPLczIk5boYeyAVFbtDDrBO/o8+P4HMHiqfZ46CH7wEZx0AQw73ZbtXtn3wUaaxmpoqHI6CqViVreJ3xizApvcO6svNsasBpo6VE0HthljdhhjGoG/AhcfT7CR5tThmaz5pJzGZn9bYcaw0Bv3Pxni07S7B2zSb6x2OgqlYlZv9vEPBoL7NYoCZVHjzFHZ1Df5WbGlpF15fVMLj7+5g5qGoOGbLrftAvrknT6OMgI1VEGDJn6lnNKbiV9ClHU6f4GI3CAihSJSWFJS0tlmEeXsMTnkpMTz19W725UvWrePX7y4kUeXb2//gmGnQ+kWqD4xPl+vaayGxirw+7vfVikVdr2Z+IuAIUHPc4F9nW1sjJlvjCkwxhTk5OT0Yljh43G7+GJBLm9sKmZ/Zd3h8uff3wvAgrd2Ulbd0PaCYWfY+98VwH3DYcsrfRluZGhuhJZG+7hJ1ytWygm9mfhXA6NFZLiIeIHLgUW9+H6OuHzaUPwGnl1dBMC+ijre3VnGxfmDqGtqad/qHzwVzrjJjv7xN8PGfzsUtYOC+/b1BK9SjojrbgMRWQjMBLJFpAi4E/AAGGMeFZEBQCGQCvhF5CZgvDHmkIjcCLwCuIEFxpiPeudjOGdIZiJnjc7mb6t3851ZI1m0fh/GwM3nnUScy8WfVn7C9WeNoH+qz/bzf/Yu+8JD+6BotbPBOyE42Ws/v1KO6DbxG2Ou6Kb+ALYbJ1TdYmDxsYV24rj2tDyu/1MhVz6+itLqBqYMTScvO4lvzRzJc2uLWPLRAb58Wl77F+VOg62vQF0FJKQ7ErcjtMWvlOP0yt0wOG98fx780mQ27K1kR0kNl06xg5dG5iTRLyWewk/Kj3zRkGn2fu+aPow0AgS38hs18SvlhG5b/KpnLp2Sy8TB6fy9cA//NdX+ABIRpuVlUrgrROIfNBUQ290z6ty+DdZJ7bp6NPEr5QRt8YfRqH7J3D53HMnxbd+nBXkZ7K2oY29FXfuNfanQbzzsea+Po3RYo/bxK+U0Tfy9bFpeJgCFu0Jc/JxbAHsLY2s8e4P28SvlNE38vWzsgBSSvO7Q3T1Dptv1enevtBOXlX/S5/H1uUbt41fKadrH38vi3C6mDssIfYI3d7q9f3KuvR9+NlwbdZc6tNfa4he3tviVcoi2+PtAwbBMNh04xKH6DvPYZY2C/KvhtBvtbefy6J/Lp7HKLknpS9U+fqUcoom/D0zLy8AY+P7C9/nLqt1tq3a5XHDJw3DB/8Gsn0JSP1h2z9HtvGy7Xc/3RNFQDfHJEJ+iLX6lHKKJvw8U5GXypYIhfLz/ED95/kNu/Mv7tPg7zFfnTYQzb7LLM77zO7toe3ND6B0GW/wjeO663gm8NzRUgTcZvCk6NbNSDtHE3we8cS7u+8Ik3r39XO78/Hhe2/gpv3x505EbFnwNMkfCkp/Cny6CP10CLc1HbtfKGHsB2MGd0NJxOYQI1VgN8amBFv8hp6NRKiZp4u9DIsJXzxjOl2cM47EVO7jnpY1t3T4AngT41jtwYyHMvhd2vwNv3N35Dg/usKOCTMuJMyKooSrQ1ZOsffxKOURH9Tjgzs+Pp6nFz2PLd/DqR58y/5pTGNUvxVZ6fJA92t5KNsPbD9l5/E+6wNb/+ybwJMLs/wf73m/b6cHtkD2q7z/M0WqshsRsm/grdne/vVIq7LTF74A4t4t7L5vEM9edSml1Aw+8siX0hrPvtVf3vnw7+Fvsidw1T8KaJ6CpziZ+cdtty7aH3kekaT25603Wk7tKOUQTv4POHJ3N5dOH8urGTyk+VH/kBh4fzLzdtuY/eh5WPQYYaKqF7W/YxD94KvjS7DYngsZqm/TjdTinUk7RxO+wy6cNocVv+PuaotAbjP0cZI+B5b+EdX+GCV+wif7jRXYY56Cp9oRw2ba+DfxYNVTxaaOHfXVuXX5RKYdo4nfYiJxkThuRxcL3duPvOMQT7Fj/s34ApZtta/n078JJs2HDP+zShYOmQNZIKNvR98EfLb8fGqt585MGXtlea8t0+UWl+pwm/ghw5alDKSqvY+nm4tAbTPgCZI6AvLNgUL79FeAPDPMcHGjxV+6BphDdRZEkkOTLm70cbPLaMu3nV6rPaeKPABecPIDcjARu+us63tpaeuQG7ji47lX40jP2+ahzIc5n+8qzRtkWPwbKd/Vl2Ecv0Kdf3hxPWVN8uzKlVN/RxB8BvHEunv3GaQzOSOArT7zHPYs3suXTDi3hpOy2JRq9STDpizBmjl3HN3OkLY/0E7yBK3UPNnspbfLYMm3xK9XnNPFHiEHpCfz9m6cxe8IAHn9rJ+c/uIJ7XwpxdW+ri34Llz1uH2eNsPeRPqQzkORLmzyUN/tsmU7NrFSf08QfQVJ8Hn535VRW/eRcLp0ymPkrtvNhUWX3L0zIgITMyB/ZE2jxV5sEaggkfm3xK9XnNPFHoOzkeO66+GQyk+L52QsfHjmhWyhZI+2CLsvug3V/ObLe77fTOzgpkOSrTAJVJATKtI9fqb6miT9Cpfo8/OzCcawvquSvq3swtcHAfCjdAsv+H7zwbdi/vn39yt/Br8Y5ewI4kORr8FFjWhO/tviV6mua+CPYxfmDmD48kwdf3UptYxezdALMvgdu/ghu2W5PAr96Z1ud3w/v/cEOp3z1jt4NuiuB/vwak0B1a4tf+/iV6nOa+COYiHDbnLGUVjfwxNu7ut7Y7YG0XDv65zM/hh1LYdvrtm7HG1C5G3Knwcf/gl1v9XrsIQVa/NX4aMSDXzza4lfKAd0mfhFZICLFIrKhk3oRkXkisk1EPhCRqUF1LSKyLnCL8sVke8fUoRmcN64/jy7fTkVtY89eNO1aBFVqAAAWDElEQVQ6SB9mJ3erKbMTuyVmw9XPQdoQeOlWqKvo1bhDaqzG4KIOO4a/KS5R+/iVckBPWvxPArO7qJ8DjA7cbgAeCaqrM8bkB24XHXOUMe6WC8ZQ3dDMfS9vxpgenOiNi4fPPwQVn8Afz4PNL0H+lXaOnzm/hOKN8OhZsPvd3g8+WEO1TfaIfepO0ha/Ug7oNvEbY1YAB7vY5GLgT8Z6F0gXkYHhClDBmAEpXH/mcBa+t5tbn/uA5pYeTGw28hy45l9QV26nd5h6rS0fOxeuW2LnAHpiru366SuNVTS6kw4/rXcl2iGe+9bBzjf7Lg6lYlw4+vgHA3uCnhcFygB8IlIoIu+KyCVheK+Y9ZO54/jeuaN5trCIH/59fc9a/kNnwNeXwhV/a79IS24BfONNe/+P62DLK70XeLCGKhpcibgEMpO81EkCfPI2/OEc+PMXoLKTGUqVUmEVjsQvIcpas9JQY0wBcCXwkIiM7HQnIjcEviQKS0pKwhBWdBERfvDZk7j5vJP417p9PP/+3p69MHM4jAnRU+dLhav+DgMmwMIr4IkLYfn99hdAyebwBt+qoZo6SSA5Po60BA/VJNpfJKPPt+sHv/GL3nlfpVQ74Uj8RcCQoOe5wD4AY0zr/Q5gGTCls50YY+YbYwqMMQU5OTlhCCs63XjOKKblZXDnvz6iqLz2+HbmS4MvPw9nfN8ufL70F/DsNfDwdHjj/2wy7kzVp7BqPhwIec6/veZGeP8Z2L+Oakkixech1RfHv5P/Gy55BK5YCDO+Bev/art9lFK9SnrSZSAiecB/jDETQtRdCNwIzAVOBeYZY6aLSAZQa4xpEJFsYCVwsTHm4+7er6CgwBQWFh7VB4klew7WMvuhFWQlx3PdmcO57JRckuPDsHxy/SEo32lX+lr3ZzjtRjjzZkjMsks9lu+CvWtg5wr4+AVoaQSXB2b9xE4aJy749GPYswpaGsCXDsUfw7bXbMu+/0T+z/UN3qwbRnZyPLWNzfzz22cE3rsSfpNvZxwdNNm+Z7tbNiRmtj33JoGE+rGpVGwSkTWBHpbut+0u8YvIQmAmkA18CtwJeACMMY+KiAC/w478qQW+aowpFJHTgccAP/aXxUPGmD/2JChN/N17Z1sp9728ifVFlUzOTeOF75yBdJEItxVX89KH+7nxnFFdbgfYC75eugVWByaBi0uA5rq2el86TPxvyL8C3v7NkSeIxW1nDW1ptAl79GftmgKjzuXyP7xLi9+QkxLPlk+ree0HZ7e9bssSeGce1JZBTam9Ny2hY3TH22sWWr8MfGl2XeLmBvul09xo7/3NNt6EjLbtqg7YL7jag/bkckKGHeYqLvs8PgVSBwW+XFyA2HsJ3CdmQb9xdj3kpOyuj6VSfSSsid8Jmvh77o9v7eTu/3zMP799OlOHZnS63Y//sZ5nC4tYfstMhmUldbrdYcbAjmV26GdlkU2w6cNg4GS7BoDL1X67yj02yWYMtxeKeZPsr4Q4X9u2wIXz3qR/qo9+KfEs3VzMqp+c13UM9ZX2CyD41vqlUHswcF9qt3N5IM5rvxRa711x7fdRXwHJ/e3CNknZ9hdG7UGo2G3PVrUuAl9ZZOPHgPHbWIyfttNXAUk5bV8Ch+/H2wXllepDR5P4w9A/oJz0pWlD+NWSzfz1vd2dJn6/37B0sz1hvm5PRc8SvwiMnGVvPdkuFG/iEUVV9c2M6hdHaoKHQ3XdTEMhYqefSEgPLDYTAYyB6k9tF1bxxrb7tU+3LSPpTYaCr9llMpP7ORuvUiFo4j/BJcfH8flJg1i0fh//87nxpPg8R2zz8f5DlFQ1ALB+TyUX5w8+Ypu+UlXfRIovjlRfHHVNLTQ2+/HGnUAzh4hAygB7G3lOW7nfb3/1FH8MG56zk+KtfcoOpR12mnPxKhXCCfQXpzpz+fQh1DW1sGj9vpD1b2wqtg3znCTWFzkwVUOAMYaq+mY7qifBfkFV1Tc5Fk9YuVyQMcyuinbZ4/DtVbYb6OlL4INn4dB+++UQrPagnTTvkTPhUOh/O6V6g7b4o0D+kHTGDkjh6ZWf8KWCIcS523+fv7GpmEm56UwblsHT735CU4sfj7vvv/Prm/w0+02gxW8T/6H6ZrKS4/s8ll6XcxJ87RV7Ydo/v27LvCn2RPegfNj3vp1Er3XKitWPw7kOzpyqYoq2+KOAiPDdc0az6UAVj63Y0a6urLqB9UUVzBqTw+Qh6TQ0+9l8wJn5cVpb9yk+Dyk+2+Y4VBclLf5QkrLhqy/BlX+HuQ/AyZfArjdtK3/PezBmLnzrHXu/5ik7IkmpPqAt/ihx4aSBLN4wkIde28KsMf3ITvGy5UA1i9bvxRg4Z2w/MhK9AKwvqmDC4LQ+j/FQvT2Zm+qLO9zVcyhauno640mAk85ve+5vsaOLknLarkOYfj1sfhE+esFeD1G+y46gcmm7TPUOTfxR5O6LJ7Bqx0Hmzmub8EwEzhvXjwmD0pDAHDnr91Rw1anD+jy+thZ/W1dPVX03I3uijct95Eif4TMhazS89SCs/gMUrbZDYmffB7mnhN5PUz28/ZAdUVRbZsvcXjtcteoAeBLtdB1JOXaUkdsTuBI7MBy19XHrcO44b+DCuOS2OFuve3B77RdW9af2QjzjD9pHkHbXh0iIso7bSedlXe3Lk2iHFxu/PTfSUG1npI3zgcdn7+MC3Yd1FfbaDHG13ZAeXvx3NBcIBh2L4OPi9thj6m+yw4r9zTY+tzcQZ+DeBJZGNX4Yf/FRvO+x0cQfRTKTvDz25am8vOEAg9MTGNkvmfwh6e1G+kzOTWP9HmfW3m1N8vbkbgx09fSUywXTvw4v/RiS+sFZP7RTXDx+DuRfZfv+Uwa0bX9oP/ztKnsVddYoe5GcuKCx3Cbq7DE22ZXvsldSN1ZBS+ALVoS2xCeB3CbQXG9vylmJ2Zr41dE7ZVgmpwzL7LR+8pB0lm3ZyjvbSzl9ZN9eddqW+INP7mriB+y4/6QcGHWenUDvzJvhzV/ByodtF9Cw0yBnLBzcCZ+8ZRP5l56BcZ8PXwyNNfYmLnvVde1B+wuipcmWJfe3LW1xd91iPtziNR2e97QsVOs5aJumWnsRnwikDrZXWrc0Br68GuyFd80N9jUJGfZiQghciOdv+8XSlWO5sPWIXynGHruGanDH2SvIXXGBK8uDbi0Ndntfqv3i7gOa+GPM5dOG8p8P9vPlP77Hjy8Yw+XThpKWeOTY/94QfHI30evG7ZLuL+KKFW4PTPivtufxKXDez2HqNfDWQ7Z1v2N5YMjoXDj9e9B/fHhj8Ca1JUmw01ZEqpwxTkdwQtPEH2MGpPl4/tun88Nn13PPS5u49+VNTBiUxumjsjhzVDanj8zG7eqdyc+CW/wiQoovTlv83ckcARfNs4+N0YnpVFho4o9BKT4Pj335FNZ8Us5b20p5Z1sZC97ayWPLdzA4PYErTx1K/pB0RuQkMTAtIWzvW9XQjAgke+1/u1SfR/v4j4YmfRUmmvhjlIhQkJdJQV4mN50HtY3NLNtcwp9W7uL+V9oWYskfks6V04dyzrh+ZB/nhVZV9U0ke+NwBX5RpCbEHR7iqZTqO5r4FQCJ3jjmThzI3IkD+fRQPduLq/lwbyXPFu7hx899AMCY/inMmTiA/5qSy9CsIydg646drqHtv1yqz8O+ijpqG5tJ9Op/RaX6iv61qSP0T/XRP9XH6aOyueEzI1hfVMnb20pZsaWE37y+lYde28rQzEQmDk4jPdGDx+0iM8nLgFQfA9J8DEzzMSwr6YjJ16rqm0gOSvxnjMrm/lc2c9Z9S7n5sydx9Yy+v7ZAqVikiV91SUTIH5JO/pB0vjNrFHsr6lj8wX7e31POhn2VVNc309jiP+JCrESvm2l5mZzUP5n0RC/xcS62flpNRpL38DbfmTWKU4dn8qslW/jZCxsormrg5vNGd79QTBRpavHz3s6DnD4yK6Y+t3KWJn51VAanJ/D1z4w4oryhuYXiQw3sr6xnX0Uda3eX8872Mt7dUUZDc9u46cuHt7/GoCAvk2euP5Vbn/uAea9vZW95HbPG5jCqXzLZyfFkJHp7bZRRJHh46TYeem0rT183nbNG61rTqm9o4ldhER/nZkhmIkMybd//JVPa5vyva2yhscVvp7IPsTaw2yX88rJJJMfH8cy7n/Dc2qLDdSKQkeglNTAE1CVtk7y5XUKcS3AfvrlwC7hcgltsWbvHIrhdtt4lYhfVMoYWv8FvwG8MfmNwid1vnNuFx223a/L7iXMJSfFxJMfHkeSNa3sc7yY5Ps5+UQX9oulOSVUD8wOT6j1bWKSJX/UZTfyq1yV43STg7nIbl0v4+UUnc/vcsWz9tJpdZTUcrGmktLqRsuqGw11JLX7DofomqhuaafGbw7dmv8EfuLeJvP19a3JvLfMbgyC4XNgvBBEk8KXRup/mFkOT348AcW7X4f10JTPJy9gBKcwYkcVZo7PJH5LeaRfOvNe30tDsZ9aYHF756ACVtU19djGdim2a+FVEiY9zM2FwmiOzh3bHGENDs5/qhmaq65upbmimpqGZmsZmqhtaKD5Uz7biaj4oquTB17bw61e3MCIniXPG9KPFGLxuFzPH9GPsgBRe31TMwvd2c8X0IVw+bShLf/sWi9bv5cun5Tn9MVUM0MSvVA+JCD6PG5/H3e01DRW1jSz5+FP+XriHp1buwhfnpqHZ3269hLysRL5/7knkpMQzbmAqzxYWaeJXfULMsUxG1MsKCgpMYWGh02EoFVa1jc0s31zC1uJqzhydTX5u+uGL2Z54eyd3/ftjzh3bj4vyB/G5SYOi+qS2Cj8RWWOMKejJttriV6qPJHrjmDNxIHNC1F0xfSjFVQ288P5eXt9UzF9W7ebBL+UzKD18U2Yo1Upb/EpFEL/f8NzaIu5c9BEet4tL8gdx7rj+TB2WQXKIEVFKtTqaFr8mfqUi0M7SGu5ZvJEVW0uob7JDYUfmJDMoPYHsZC85yfFkJXvpl+KjX0o8U4ZmkODteuSUim7a1aPUCW54dhLzrymgrrGFd3eW8cGeSj7aV3l4HqWS6gYagy6My0mJ5/vnjuas0e0X10n0xpGR6CHOHXr93qYWPzUNzfiNnS7b08l2Krr0KPGLyALgc0CxMWZCiHoBfgPMBWqBrxhj1gbqrgV+Ftj0F8aYp8IRuFKxIMHrZtaYfswa036dXmMMVQ3NlFY1sKushkeWbednL2zodD+tF6KBXcuq9Zd+x8sSPG5BQq0128l55s5OP4dcajewdWvd4VV2JegdQ9S1vuaIsiPeS0Ls+8j37rifjjHbbdrHagwYTOCiv9CfueM+uvrMHT+HwX4JZyR6+fd3z+z8DcKkpy3+J4HfAX/qpH4OMDpwOxV4BDhVRDKBO4EC7GdbIyKLjDHlxxO0UrFOREj1eUj1eRiRk8ysMf1YuaOMA5Vt6+YaY0cSldU0Hv510DGpxblcgYVx7OypdU0tR7xXZ4nOEKKi021b92Xa7dO0e9y+jqDXtL0+9Lam3es61AW9Jnj7UPvDhIgVm5wPf0EFjmHwl0LHfYT8zO0+E0eUeVzSZxfw9SjxG2NWiEheF5tcDPzJ2E/4roiki8hAYCbwqjHmIICIvArMBhYeT9BKqfZEpM/XUFYnrnB16A0G9gQ9LwqUdVZ+BBG5QUQKRaSwpKQkTGEppZTqKFyJP1RXn+mi/MhCY+YbYwqMMQU5OTpZlVJK9ZZwJf4iYEjQ81xgXxflSimlHBKuxL8IuEasGUClMWY/8ApwvohkiEgGcH6gTCmllEN6OpxzIfZEbbaIFGFH6ngAjDGPAouxQzm3YYdzfjVQd1BE7gZWB3b1v60nepVSSjmjp6N6ruim3gDf6aRuAbDg6ENTSinVG/QyPaWUijGa+JVSKsZE5CRtIlICfHKML88GSsMYTm/QGI9fpMcHGmO4aIw9M8wY06Ox8BGZ+I+HiBT2dIY6p2iMxy/S4wONMVw0xvDTrh6llIoxmviVUirGRGPin+90AD2gMR6/SI8PNMZw0RjDLOr6+JVSSnUtGlv8SimluhA1iV9EZovIZhHZJiK3OR0PgIgMEZGlIrJRRD4Ske8HyjNF5FUR2Rq4z4iAWN0i8r6I/CfwfLiIrArE+DcR8TocX7qI/ENENgWO52mRdhxF5ObAv/MGEVkoIj6nj6OILBCRYhHZEFQW8rgF5tqaF/gb+kBEpjoY4/2Bf+sPROR5EUkPqrs9EONmEbnAifiC6n4kIkZEsgPPHTmGRysqEr+IuIGHsSuBjQeuEJHxzkYFQDPwQ2PMOGAG8J1AXLcBrxtjRgOvB5477fvAxqDn9wEPBmIsB65zJKo2vwFeNsaMBSZjY42Y4ygig4HvAQWB5UndwOU4fxyfxC5+FKyz4xa8kt4N2JX0nIrxVWCCMWYSsAW4HSDw93M5cHLgNb8P/P33dXyIyBDgs8DuoGKnjuHRMcac8DfgNOCVoOe3A7c7HVeIOP+F/Y+yGRgYKBsIbHY4rlxsAjgH+A92HYVSIC7U8XUgvlRgJ4FzUkHlEXMcaVt0KBM7B9Z/gAsi4TgCecCG7o4b8BhwRajt+jrGDnWXAn8OPG73t42d7fc0J+ID/oFthOwCsp0+hkdzi4oWP0ex0pdTAktXTgFWAf2NnbaawH2/zl/ZJx4Cfgz4A8+zgApjTHPgudPHcwRQAjwR6I56XESSiKDjaIzZCzyAbf3tByqBNUTWcWzV2XGL1L+jrwEvBR5HRIwichGw1xizvkNVRMTXnWhJ/D1e6csJIpIMPAfcZIw55HQ8wUTkc0CxMWZNcHGITZ08nnHAVOARY8wUoIbI6B47LNBPfjEwHBgEJGF/9ncUMf8vQ4i0f3dE5KfYLtM/txaF2KxPYxSRROCnwB2hqkOURdy/ebQk/ohd6UtEPNik/2djzD8DxZ+KXYyewH2xU/EBZwAXicgu4K/Y7p6HgHQRaZ222+njWQQUGWNWBZ7/A/tFEEnH8TxgpzGmxBjTBPwTOJ3IOo6tOjtuEfV3JCLXAp8DrjKBfhMiI8aR2C/49YG/m1xgrYgMiJD4uhUtiX81MDowgsKLPfmzyOGYEBEB/ghsNMb8OqhqEXBt4PG12L5/RxhjbjfG5Bpj8rDH7Q1jzFXAUuALgc2cjvEAsEdExgSKzgU+JoKOI7aLZ4aIJAb+3VtjjJjjGKSz49bZSnp9TkRmA7cCFxljaoOqFgGXi0i8iAzHnkR9ry9jM8Z8aIzpZ4zJC/zdFAFTA/9PI+YYdsnpkwxhPPkyF3v2fzvwU6fjCcR0JvZn3gfAusBtLrYP/XVga+A+0+lYA/HOBP4TeDwC+we1Dfg7EO9wbPlAYeBYvgBkRNpxBO4CNgEbgKeBeKePI7AQe86hCZugruvsuGG7KR4O/A19iB2h5FSM27B95a1/N48Gbf/TQIybgTlOxNehfhdtJ3cdOYZHe9Mrd5VSKsZES1ePUkqpHtLEr5RSMUYTv1JKxRhN/EopFWM08SulVIzRxK+UUjFGE79SSsUYTfxKKRVj/j+YeyKT721HHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_losses, label = \"train loss\")\n",
    "plt.plot(test_losses, label = \"test loss\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f168022f390>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VPW5+PHPk8m+J4R9MYGCIEsAwyYuKIuoFXfFpS6tYluX9rZ6C9Vaf3rt9aq3i7fW1rZoa61otQpa3FBcqqCAIrKIhEUIgZB932bm+f1xJmEICRlCyCQzz/v1mtfMOed7Zp4cyDPfPOd7vkdUFWOMMeEhItgBGGOM6TqW9I0xJoxY0jfGmDBiSd8YY8KIJX1jjAkjlvSNMSaMWNI3xpgwYknfGGPCiCV9Y4wJI5HBDqCljIwMzczMDHYYxhjTo6xbt65IVXu31y6gpC8ic4HfAC7gT6r6YIvtvwLO9C3GA31UNdW3zQN84du2W1XnHemzMjMzWbt2bSBhGWOM8RGRrwNp127SFxEX8BgwG8gD1ojIMlXd3NRGVf/Dr/1twAS/t6hV1fGBBm6MMeb4CaSmPxnIVdUdqtoALAEuOEL7K4FnOyM4Y4wxnSuQpD8Q2OO3nOdbdxgROQHIAt7xWx0rImtFZLWIXNjhSI0xxhyzQGr60sq6tuZjng+8oKoev3VDVDVfRIYC74jIF6q6/ZAPEFkALAAYMmRIACEZY4zpiEB6+nnAYL/lQUB+G23n06K0o6r5vucdwLscWu9vavOEquaoak7v3u2efDbGGNNBgST9NcBwEckSkWicxL6sZSMRORFIA1b5rUsTkRjf6wxgOrC55b7GGGO6RrvlHVV1i8itwBs4QzYXq+omEbkPWKuqTV8AVwJL9NBbcY0C/iAiXpwvmAf9R/0YY4zpWtLdbpeYk5OjNk7fmKNTWFnP0vV7qahtDHYo5hj0S4njqikdO68pIutUNae9dt3uilxjjENV2Vdex9b9lbi9Bztnbo+XPaU17C6pwe1RKuoaWbH5AA0eL9LasAvTY4wfnNrhpB8oS/rGdDP7y+t4evUu/vnpXvaV17XZLjU+ipjICFwiXDFpMDdMz2Ro78QujNT0RJb0jelC9W4PJdUN9E+JO2T9p7tLueuljRRX1VNUVY8CM0f24btnDGP0gGRio1zNbSNEGJgWR0pcVBdHb0KBJX1jusiekhpufnodW/ZXcEH2AH44awSZGQkA/PmDneSV1nDumP70TYnl0omDGNIrPsgRm1BkSd+Y48zt8bJ8437uXbaJRo+X+ZOG8NJneazYcoB3fnwGMZEu3tpSwFWTh3DvvNHBDteEOEv6xnSymgY3G/LK+XJfBTuLqlmx5QB7y2o5sW8Sj18zkaG9E7nxtCzO/tX7/HZlLiP7JdPg9nLJxEHBDt2EAUv6ptuobfCwt6z2qPapaXCzs6iagoo6unL0seIMk9xZVE1VnRtFqW30UFrdyL7yWpoG2yTGRDJ+cCr3zhvNWSP74IpwhtcM653I5ZMG8+wnuxmSHs/wPomMGZjcdT+ACVuW9M1x8+X+CkqqGg6uEMgelEpCjPPfzu3x0uhR4qJdvLW5gJ++9AWFlfVBivboxUZFkNkrgbT4aEDonRjJiD5JDEqLY8KQNEYPTKZ3YgzSxjjK288azgvr8theWM3Cc0a22c6YzmRJ3wTE61W2FlTS4PYeXKdKeW0j5bWNeP262dX1Hl78NI/Pdpcd9j7fnp7FPeefBMD1T67h37lF9E6KobCynlH9k1l0zkgiXYHfxTMmMoKsjAT6p8Q296K7Smyki4hj+Mx+KbHccEomT364iwvGD+jEyIxpmyV906oDFXW8umEfjR4vxdUNvPp5PvlHGDPe0gm94vn5+SdxUv+DJYs7XvicfeUHyzfbC6sY1T+ZUf2TGNE3iW9PzyI6Mrxu23zn2SdyzdQTDhvCaczxYknfHKKu0cNfV+3i0bdzqap3A+CKEE4fnsGP55xIWsLBseGCkBwXSUpcNJF+Pd6mceQte979kmMprTlY7imtaeCb4/pz13knHd8fqhuLdEUwON2GZpquY0k/zOwpqWHVjmLOHzeAuGgXBRV1rN5RjFeVHYXV/P3j3RRXN3DWyD789NyRDEh1kndMpKv9N29HSlw0eaU1gPPlUtfoJTU++pjf1xgTOEv6YeZXb33FPz/by8NvbOXkIWms2FLQPK+LiHMV6LdPzeKUYRmd/tlp8VFs3OtMCNbU40+zpN++mhLY/g7s/RQGTIBvzIT49GBHZXooS/ohZltBJQ0eL6MHpBy2TVVZvaOYCUNSiXZFsGpHMddOy+SSkweSEB1JQkwkvZNijltsaQnRzcm+tNpJ/qnxNpUAAPWVsO1NGDINkvrDgS2w7Q346k3YsxrUCxGR4HU7z2f/AqbcHOyoTQ9kST+E/HtbETf9dS1eVZ65cQo5mYf2BvNKa8kvr+O7M4Zx7bTMLo8vNT6KereX2gYPZbUNzesM8Pb98MkfnNfxvaCm2Hndbxyc9mMYfjYMGA/7Pof3H4HX/hPKdsOYizs/FgWKvoKvXoeS7Qdj+sZsGDwZIo691GfaEJUAfUYe14+wpB8iVm49wM1Pr2NoRgINbi/ffmoNz908jVF+o2dW7XASyZSsXkGJMTXOKeWU1TZQVuP09MOuvPP1R7DnY5h2G7h8v36VBfDpX2DUPBh4stPLP2EaDJ8DyS2Gcg7KgfnPwGs/gVW/dR7HS2I/54tGIqBkJ7x51/H7LOMYmAM3vX1cP8KSfghQVe5ZupGhGQksWTCVyjo3l/7+I8599APGD07lW1NP4OKJg/h4RwnpCdEM7xOc6XfTfL360urG8Kzpf/4cLL0FvI1O8r/0SYhJhFX/B54GmHUv9BrW/vtEuODch2HsZVB3+LUQnSKpH/QdCxF+Q2hLv4bCL4/P5xlH7OFl2c5mST8EfLG3nD0ltTx06ThS46NJjY/mxe+dwgvr8njti/38+B+f840+iXy8s5jJmenHdEHRsWgaqVNWc7CnHxblHVX49y/h7fsg8zQYeR68cRf88Uwnca9Z7DwHkvCbiMCQKccv5takneA8TI8W0JUwIjJXRLaKSK6ILGxl+69EZL3v8ZWIlPltu05Etvke13Vm8Mbxry/2ERkhzDmpb/O6QWnx/HDWCP7xvWn0Tozh9mc/I6+0lilDgzfqo2mMf2lNI6XVDcRFuQ6ZJz4kedzwrx85CX/sZXDNizD1e3DV8xAVDysfAHctnPqjYEdqwkS7PX0RcQGPAbOBPGCNiCzzv8G5qv6HX/vbgAm+1+nAz4EcnNND63z7lnbqTxHGVJXlX+xj+jcyWh3znhwbxT3nn8Stf/8MgKlDg1PPh0Nr+qU1jc3lnpC1bwMsv9MZfXPqf8BZ9xwslwyf5Twq9zsnbY/zyTtjmgRS3pkM5KrqDgARWQJcAGxuo/2VOIke4GzgLVUt8e37FjAXePZYgjaw5JPdAJw0IJk9JbXcdubwNtueN7Y//xiRx+Z9FZzYN6mrQoSKfZCQAS4nuTeVcspqGimraSAlFOr51UVOrRuFgk3OsMuyr52SzoHNEJcGF/4exl/Z+v5J/ZyHMV0kkKQ/ENjjt5wHtFpMFJETgCzgnSPsO7CV/RYACwCGDDm+NwUOBUVV9fxs6UYaPUpybKRT2hndt832IsLvrzmZ8trGwOv55Xvhw99AY83h26IT4dQfHjlZNZ20HDAernwOEnoRG+UiLspFaXUDZbWt9PRV6dL5kY+Fpx5WPQYf/O+hxyh5EPQb69TcvzHL6eHHpQYvTmNaCCTpt5Yl2vrNnA+8oKqeo9lXVZ8AngDIycnpIb/1wfPSp3tp9Cg/mDmcxR/u5KyRfdqdziAu2kVcdID187oKeOYyKN4G8a1cmVtdCLtXwQ3LISIKtiw7dBRJ8Q5Y/Rj0Hw/7v4A/z4Z5j8LgqaTFRzk1/ZoGRvXzmz/e3QDPXeNckNSTjDofxl/jDGtMGQh9TnISvjHdVCBJPw8Y7Lc8CMhvo+184JYW+85ose+7gYdnWlJVlqzZzcQhqfzH7BHcdPpQXB1JMl4v7N8AXs/h2979b2do3jUvwrAzD9++9XVYciU8Ox8q8qE49/A2Yy+HC34L+euddk+dB7GpzIu8idzasymraWRe5RJYvgTO+E946+dOwp98s3MhUE9wwjTIOj3YURhzVAJJ+muA4SKSBezFSexXtWwkIicCacAqv9VvAL8QkTTf8hxg0TFFHOY+3V3K9sJqHrpkHODcmalDlt4Cn/+97e3n/6b1hA9w4lxnGoDXF0L6MKd8M3Diwe3iggRf4h4yBX7wOexYCa/8gOneT/mk+kzKahqYXPov2J8Hn/7VGcFyxk/gzJ927OcxxgSk3Yyhqm4RuRUngbuAxaq6SUTuA9aq6jJf0yuBJaoHi7KqWiIi9+N8cQDc13RS1xydlz/bS+6BKtZ+XUJCtIvzxvU/tEHBJqfkour00re9CY21Tl15zCXOc9NfBFtecRL+pJucqz5bSsg4NIm3Zsp3nXli+oyCyHbm64lNhpMugPcfIaWylt0lzu0E47xVzudLBCQPhBnWHzDmeAuom6iqy4HlLdbd02L53jb2XQws7mB8BlixuYAfPre+efm6aSc033Kw2Ss/hLxPnNdR8TB0hvO89TX4/FkYdpZzUjEi0mnbPxvm/nfzyJqjJuKcpD0asSkkV5ZTVFUPKNHuSmdumZk/61gMxpijZlfkdnMl1Q0s/OcXjOyXxMu3TKfe7SU5tpV/ttKdMG4+zLnfuZS7qfftaYQ1f4KV/w1/Od9Z54qBi/7Q8YTfUbEpJHid00EJ1BGhni657NwYc5Al/W5qU345m/ZW8PL6vZTXNvD0dyYT29YVrA3Vzoia3iMgsc+h21xRzhWgYy+Hveucdb2GHd0l/50lLpV4byUAydQ0rzPGdB1L+t3MhrwyHn5jKx9sKwIgyiXcfd5Jh8yWeZgy36UQqUeYFyWhF4xopX7flWJTiXE7ST9Fqn3rrKdvTFeypN+NfLm/got/9xFJsZEsOmckc8f0Y2BqHJGudqZIKnOuziW1m1/YFptKpKeWSNwkY0nfmGCwpN9NqCo/X7qJxNhIVvzoDHoltjMiZt8GJ2GmneBc9g9H7ul3B74En0wNqRFNSd/KO8Z0pYBm2TTH3ysb9vHxzhLuPPvE9hM+wLNXOuPkwUn6kbGH1/O7G1/9PkWq6Rtd76yznr4xXcp6+kHm9nh5c3MB//XqZsYMTGb+pABKNJUFUJHn3IwDnPJOyuDuf/l/c0+/mr5RddCAncg1potZ0g+i0uoGLnn8I3YUVTM4PY4HLx6HK5AJ0fb5xuxXFTgzWZbt7v71fGhO+ilSTUZkrZP0Y45wgtoY0+ks6QfR3z/ZzY6ian4zfzzfHDcgsIQPkP/Zoa9Lv3YmN+vufPX7vtH1pLtqISbFbrJtTBezmn6QuD1enln9NacM68UF4wceOeF7PbDqd7Dvc2c5f73Ts5cI+PpDqC3pGbex8/X0v5Hkpl90vdXzjQkCS/pB8tbmAvLL67julMwjN2yocaYcfmMRvOY7cbtvvTPvTcaJsNk39VFPKO/46vffyUlnTC+FOEv6xnQ1K+90obpGD39b/TWD0uL48793MjA1jlmj2r75CV4v/O1i2L0ahpwCuz+Crz+Cyn1OOUcinHl1oPsP1wRnhJErmsiGCqgrt+GaxgSBJf0udP+rm3nm493NywvPGXnkss6Wpc7Mmec/6sxG+auT4LWfONsGTHDq4c1Jvwf09EWcRF9X5iT99KxgR2RM2LGk30Xe3LSfZz7ezXdOzeLcsf3ZXljFvOwBhzfc+YFz96W4NHj/Eeg1HCZc4yT4YTMh9y1AfLfk81XnIuMgoXeX/jwdFpviJPy6MuvpGxMElvS7QFFVPT95cQNjBibzk7kjiY6M4OQT0g5vWLAZ/vJNp1Qz+SYo2OjcVLtphMuEq52knzECYhIPJv7UId1/jH6T2BSo9fX07USuMV3Okn4XeOXzfEprGnl2wVSiI49w7nz9M849Zxuq4c27IS0Txl52cPuJ5zr3rB082VmOjnfmo08d3OrbdUtxqVB1ABqq7MIsY4LAkn4XWLGlgG/0SWRkvxYXIlUWwB9Og3P+B0Z+EzY859yKcPZ9sOx2Z0pkl98/UWQMLFh56AVNVz3nfFH0FLEpkLf24GtjTJcKaMimiMwVka0ikisiC9toc7mIbBaRTSLyd7/1HhFZ73ssa23fUPHp7lJu+uta6hoP3my8oq6Rj3eUtD5K56vXnatqX/kBfPoXZ0788VdD+lC4/lUYed7h+6QOObSHnNTv4P1oe4KmE7lNr40xXardnr6IuIDHgNlAHrBGRJap6ma/NsNxbng+XVVLRcR/5q9aVe0Bl4seG69XufuljWzeV8HaXaWcOjwDgPe2FuL2KrNGtTIZWu5bEN/LGYv/rzsgoQ98Y3YXR97F/Hv31tM3pssF0tOfDOSq6g5VbQCWABe0aHMT8JiqlgKo6oHODbP7W/r5XjbvqwBg9Y7i5vVvbykgPSGaCUNanLj1NMKO95yyzuz7AIXsKw4t54Qi/79SrKZvTJcLJMMMBPb4LecBU1q0GQEgIh8CLuBeVX3dty1WRNYCbuBBVX352ELufuoaPTzyxleMHpBMpCuCVb6k7/Z4Wbm1kFmj+h4+Hn/PJ1BfAcNnw4nnQXQCnHhOEKLvYtbTNyaoAkn6rY0F1FbeZzgwAxgEfCAiY1S1DBiiqvkiMhR4R0S+UNXth3yAyAJgAcCQIT3gIiM/dY0eFr64gb1ltfzPJeP4cHsRf3x/B9X1btbvKaO8trHt0k5EJGSdARERznDMcOBfx7ekb0yXC6S8kwf4jwkcBOS30mapqjaq6k5gK86XAKqa73veAbwLTGj5Aar6hKrmqGpO79495CIjoLCyniv+sIqX1+fzo9kjOHV4BtOG9sLtVdZ9Xcrv39tORmI0M05sLemvgMFTITbMphY+pKdv5R1julogSX8NMFxEskQkGpgPtByF8zJwJoCIZOCUe3aISJqIxPitnw5sJkT88YMdbN5XwR++dTK3zxwOQE5mGpERwhPv7+CDbUXceNpQ4qJbTB9cuR/2fwHDZwUh6iBrSvoRURAVF9xYjAlD7ZZ3VNUtIrcCb+DU6xer6iYRuQ9Yq6rLfNvmiMhmwAPcqarFInIK8AcR8eJ8wTzoP+qnp/t4RzEThqRx9uh+zevioyPJHpzKv3OLSI2P4pqprUyElvu28/yNMEz6TSdv41J7zlXExoSQgIaKqOpyYHmLdff4vVbgR76Hf5uPgLHHHmb3U1XvZmN+BbfMGHbYtmlDe7Hu61K+PT2LxJhWDnHuW5DUH/qO6YJIu5mmko7V840JihAfH3j8rPu6FI9XmZx1+IVRF00cyI6iqtbnyve4Yfs7MOr88OzpNl1NbPV8Y4LCkn4HfbKzmMgIYeIJhyevYb0T+d3VJ7e+4951zmRj4VjaAec6hOgk6+kbEyR256wAeL2K2+M9ZN0nO0sYMzCF+Oij/N7MfQvEBUPP7MQIe5j4dOdKZGNMl7OkH4BfvvUVFz/+UfNyXaOHz/eUMyUr/cg7qkLxducet01yVzizZIbz1agX/R7O+EmwozAmLFl5JwBf7q9g495yGtxeoiMj+Gx3GQ0eL5PbS/pfvQHPXuHcECXrdKeskf8ZnHV31wTeXZ1wSrAjMCZsWdIPQFFVA16FvWW1ZGUksHZXCSKQk9lO0t/+NkTFw4i5zr1tvW5IHwajL+6awI0xpgVL+gEoqqoHYHdJDVkZCWwtqGRwWjwpcb557Hf9G15fBOp1bls4/+/ODU6+/sgp5Vz0+yBGb4wxB1lNPwDFVQ2Ak/QBdhRWM7R3wsEG//41lO12Ev6Olc48+TUlULAJTpgejJCNMaZVlvTbUV3vptZ3U5Q9JTV4vcqOoiqG9U50GlTkO2WcSd+Ba16ExL6w6Z+w52NALekbY7oVK++0o6mXD7C7uIb88lrqGr0Hk/7nS5yyzvirnRuYn3Shcxes+F7gioaBbYzXN8aYILCefjsKffX86MgIdpfUsL2wGsAp76g6NzMfMg16+aZjGHMxuOvg06dhYA5ExQYrdGOMOYwl/XYU+5L+2IEp7CmpYfuBKsC56pY9n0BxrtPLbzJoMiQPBPVAppV2jDHdiyX9dhT5yjsTBqdSWe9m3e5SkmMjyUiMhg3PQWQcjL7w4A4RETD6Iue1jUc3xnQzVtNvR1NP37nH7U7e/6qQYb0TEa8HNi+FEWdDTNKhO035rnMVrp3ENcZ0M5b021FUVU9SbCTD+zonbivr3E5pZ9f7UFPk1PBbSh0M5zzYxZEaY0z7rLzTjqLqBnonxjA4Lb553bA+CbDxnxCdCMPnBDE6Y4w5Opb021FUWU+vxGjiol30TooBYFh6NGx5BU481275Z4zpUSzpt6O4uoGMRCfZD0l3evtj6z+FurLWSzvGGNONBZT0RWSuiGwVkVwRWdhGm8tFZLOIbBKRv/utv05Etvke13VW4F2lqMrp6YOT9F0RQt/97zulnWFnBTk6Y4w5Ou2eyBURF/AYMBvIA9aIyDL/G5yLyHBgETBdVUtFpI9vfTrwcyAHUGCdb9/Szv9ROs9jK3MpqKjjZ988ibKaRsboNlj/BddMncvoAcm4Nv8PDJgAkTHBDtUYY45KID39yUCuqu5Q1QZgCXBBizY3AY81JXNVPeBbfzbwlqqW+La9BcztnNCPnzc3F/DsJ7ubJ1ibWvAcvPIDTu4fy41TB0DBRptewRjTIwWS9AcCe/yW83zr/I0ARojIhyKyWkTmHsW+3c7e0loaPcrS9fkAJLuLwNMAe1bD/g3OvPiDcoIcpTHGHL1AxulLK+u0lfcZDswABgEfiMiYAPdFRBYACwCGDBkSQEjHT12jp3n+/BfX5QEQX1/kbNzxHiT1c15bT98Y0wMF0tPPAwb7LQ8C8ltps1RVG1V1J7AV50sgkH1R1SdUNUdVc3r37n008Xe6/LJaAGIiI9hbVgso0bUFzsYd78LedZA0AJIHBC1GY4zpqECS/hpguIhkiUg0MB9Y1qLNy8CZACKSgVPu2QG8AcwRkTQRSQPm+NZ1W3t9Sf/8bCepJ1FLhLvWuc/tvs9h5/swcGIwQzTGmA5rN+mrqhu4FSdZbwGeV9VNInKfiMzzNXsDKBaRzcBK4E5VLVbVEuB+nC+ONcB9vnXd1t5SJ+lfNcUpMw2MLHc2jL4YUKgqsHq+MabHCmjuHVVdDixvse4ev9cK/Mj3aLnvYmDxsYXZdfLLaokQZyrlsQNTyKzYBo3AyPOcG6Y0Vjvz5BtjTA9kV+S2kFdWS7/kWKJcESw6dyTXj/PdBCV1iG+qZIEB44MaozHGdJTNstnC3tJaBqY58+mcMiwD9jsjeUjqB6f9GLJOP3wqZWOM6SEs6bewt6yWnBPSDq6o3A9RCU6iP2Ga8zDGmB7Kyjt+PF5lf3ldc08fcJJ+09h8Y4zp4Szp+ymoqMPtVQaktkz6/YMXlDHGdCJL+n6axugPPCTp74OkvkGKyBhjOpclfT9NV+MOairvqG9cvvX0jTEhwpK+nzzfhVnN5Z36CmissZq+MSZkWNL3s7eslvSEaOKjfYOaKvc7z4mW9I0xocGSvp89JTUt6vm+pG89fWNMiLBx+j51jR7W7CrhipzBkLsCKgsgwnd4rKZvjAkRlvR9Pswtoq7Ry8xRfWHlD2DvWhg8xdloo3eMMSHCyjs+K7YcIDEmkimZKVCwCcQFez52boBu0y4YY0KEJX3A61Xe3lLA6SMyiCnfCe5amPNfkD4M0jKDHZ4xxnQaK+8AG/PLOVBZz6xRfWHfKmfl0DNg/JVQXxXc4IwxphNZ0gdWbC4gQuDME/vAhxvAFQMZI8AV5dwxyxhjQoSVd4CPthczfnAqaQnRsH8D9BnlJHxjjAkxlvSB0poG+qfEOdMu7P8C+o0NdkjGGHNcBJT0RWSuiGwVkVwRWdjK9utFpFBE1vseN/pt8/itb3lD9W6hqt5NUmwkVORDTTH0zw52SMYYc1y0W9MXERfwGDAbyAPWiMgyVd3coulzqnprK29Rq6rd+v6CVXVuEmMinV4+WE/fGBOyAunpTwZyVXWHqjYAS4ALjm9YXcfjVaobPCTGRjr1fAT6jg52WMYYc1wEkvQHAnv8lvN861q6REQ2iMgLIjLYb32siKwVkdUicmFrHyAiC3xt1hYWFgYefSeoqndzq+slrl1/Fax6DNKH2sVYxpiQFUjSl1bWaYvlV4BMVR0HrAD+4rdtiKrmAFcBvxaRYYe9meoTqpqjqjm9e/cOMPTOUVXv5jzXx8Q2lkPmqXD6nV36+cYY05UCGaefB/j33AcB+f4NVLXYb/GPwP/4bcv3Pe8QkXeBCcD2Dsbb6arq3CRQS2nfacTPfzrY4RhjzHEVSE9/DTBcRLJEJBqYDxwyCkdE/KehnAds8a1PE5EY3+sMYDrQ8gRwUFXWNZIotYiVdIwxYaDdnr6qukXkVuANwAUsVtVNInIfsFZVlwG3i8g8wA2UANf7dh8F/EFEvDhfMA+2MuonqCrr3SRQR0OsJX1jTOgLaBoGVV0OLG+x7h6/14uARa3s9xHQrcc/1lTXECNuouJTgh2KMcYcd2F/RW5dTTkAUXHJQY7EGGOOv7BP+g3VZQBEJ1jSN8aEvrBP+o01FQBEW3nHGBMGwj7pu+sqAYiwE7nGmDAQ9klfa52ePjFW3jHGhD5L+k13xopODG4gxhjTBcI+6VPvlHeIsaRvjAl9YZ/0pdHX07crco0xYSDsk76rsdp5YeUdY0wYCPukH+Wuoj4iDiJcwQ7FGGOOO0v6nhoaXPHBDsMYY7pEWCd9j1eJ81bTGJkQ7FCMMaZLhHXSr/LNsOmJtHq+MSY8hH3ST5RaNMp6+saY8BDeSb/OTSJ1qA3XNMaEibBO+pV1jSRQaxdmGWPCRngnfV95JyLW5t0xxoSHgJK+iMwVka0ikisiC1vZfr2IFIrIet/jRr9t14nINt/jus4M/lg1lXci46y8Y4wJD+3eLlFEXMBjwGwgD1gjIstaudftc6p6a4s16UwmAAAZcUlEQVR904GfAzmAAut8+5Z2SvTHqKa2lhhppCHO5tI3xoSHQHr6k4FcVd2hqg3AEuCCAN//bOAtVS3xJfq3gLkdC7Xz1TXdNSveyjvGmPAQSNIfCOzxW87zrWvpEhHZICIviMjgo9w3KBqrnfvjWtI3xoSLQJK+tLJOWyy/AmSq6jhgBfCXo9gXEVkgImtFZG1hYWEAIXWOxlpnWmWxE7nGmDARSNLPAwb7LQ8C8v0bqGqxqtb7Fv8InBzovr79n1DVHFXN6d27d6CxHzOv71aJNsOmMSZcBJL01wDDRSRLRKKB+cAy/wYi0t9vcR6wxff6DWCOiKSJSBowx7cuqBo9XlQVrbNbJRpjwku7o3dU1S0it+IkaxewWFU3ich9wFpVXQbcLiLzADdQAlzv27dERO7H+eIAuE9VS47Dz3FU5j+xmnq3h1PqnBO5dnGWMSZctJv0AVR1ObC8xbp7/F4vAha1se9iYPExxNipPF7l8z1luL3KaFcJRGF3zTLGhI2Akn4oKaio4yTNZfak0URt80IdVtM3xoSNsEv6eaW1PBb1KAlFI0mbMh3ew5K+MSZshN3cO3uKq+kjZaQVrEYq90FUPLjC7rvPGBOmwi7bFRQVEyON4AW+fNV6+caYsBJ2Pf3y4n0HF2qK7SSuMSashF3Sry4tcF4k+S4tsOGaxpgwEtJJ/2+rv2ZTfvkh6+orDjgvxl/tPNuFWcaYMBKySd/t8XL3yxu54g+rWbOrpHmd1BQ7DcZcApFxVtM3xoSVkE36tY0eAKob3Hzrzx+zZlcJ+yvqSFVfzz9lEJz7EEy68QjvYowxoSXkk/4PZg4nLT6aR9/eRl5pLelSiTci2jmBO/FaGD4ryJEaY0zXCdmkX9fgBWBQWjyXnjyID3OLWPd1KelU4o3rBdLarM/GGBPaQjbpN/X046JcXDB+IF6FJz/cSS+pICIxI8jRGWNMcIRs0q9pcAMQH+3iG30SGTcohaKqBvpGVhGRYEnfGBOeQjbpN/X0Y6NcAFw43rlLY0ZEFcT3ClpcxhgTTCGb9OsaPVzjeou++94BVb6Z3Z8IgRStAOvpG2PCVMgm/dq6ev4r6kmGrrgR/nYxfbSE31x6EnHeaoi3pG+MCU8hm/QbaqsAqBswFXZ9CO8/zPnDY5yNCVbeMcaEp5CdZdNTXw1A40kXExsTDXs/heoiZ6PV9I0xYSqgnr6IzBWRrSKSKyILj9DuUhFREcnxLWeKSK2IrPc9ft9ZgbfHXef09KNiE2DABDiwGSr3OxutvGOMCVPt9vRFxAU8BswG8oA1IrJMVTe3aJcE3A583OIttqvq+E6KN2Du+hoAouISIXY8eBpg1/vORjuRa4wJU4H09CcDuaq6Q1UbgCXABa20ux94COeus0Hn9ZV3XNEJMMD3nZP7tvNsPX1jTJgKJOkPBPb4Lef51jUTkQnAYFV9tZX9s0TkMxF5T0RO63ioR8fb4PT0iYqHtCyITXFKPBIBcaldFYYxxnQrgST91iap0eaNIhHAr4Aft9JuHzBEVScAPwL+LiKHTWAvIgtEZK2IrC0sLAws8naor6dPVJwzz05/X28/Lg0iXJ3yGcYY09MEkvTzgMF+y4OAfL/lJGAM8K6I7AKmAstEJEdV61W1GEBV1wHbgREtP0BVn1DVHFXN6d27d8d+kpYaa53n6ATnuanEY6UdY0wYCyTprwGGi0iWiEQD84FlTRtVtVxVM1Q1U1UzgdXAPFVdKyK9fSeCEZGhwHBgR6f/FK0Qd1N5J855HjDBebaTuMaYMNbu6B1VdYvIrcAbgAtYrKqbROQ+YK2qLjvC7qcD94mIG/AA31XVks4IvD3i9vX0o3w9/abyjo3RN8aEsYAuzlLV5cDyFuvuaaPtDL/XLwIvHkN8HRbR2KKnn5bp3Aw9dUgwwjHGmG4hZK/IjWju6fuSvgjc+DbE2o3QjTHhK2STfqSnlnqJJcb/DlkpA9vewRhjwkDITrgW6a2jMSI22GEYY0y3ErJJP8pbh9tlSd8YY/yFbNKP9tbhdsUFOwxjjOlWQjLpN3q8xGo9Hkv6xhhziJBM+rWNHuKkHm+kJX1jjPEXkkm/rsFDHJb0jTGmpZBM+rWNHuJoQKPigx2KMcZ0KyGZ9GsanPJO84VZxhhjgBC9OKu20UMf6qlrmmHTGNOuxsZG8vLyqKvrFvdBMm2IjY1l0KBBREVFdWj/0En6NSXw5t0w7grqvKOJp576aCvvGBOovLw8kpKSyMzMRKS122iYYFNViouLycvLIysrq0PvETrlnQgXrH8G9m+gtqGROGnAFWM9fWMCVVdXR69evSzhd2MiQq9evY7pr7HQ6enHJENEFFpdTH2M7/64MdbTN+ZoWMLv/o713yhkevoFlfUUehPZuXs3jXVNST8xyFEZYwJVVlbG7373uw7te+6551JWVtbJEYWmkEn66QnRFHsT8VYX4q6vAiAy1so7xvQUR0r6Ho/niPsuX76c1NTU4xHWMVFVvF5vsMM4RMgk/ShXBNWRKUTUluKuc26gEh1nPX1jeoqFCxeyfft2xo8fz5133sm7777LmWeeyVVXXcXYsWMBuPDCCzn55JMZPXo0TzzxRPO+mZmZFBUVsWvXLkaNGsVNN93E6NGjmTNnDrW1tYd91iuvvMKUKVOYMGECs2bNoqCgAICqqipuuOEGxo4dy7hx43jxReceUK+//joTJ04kOzubmTNnAnDvvffyyCOPNL/nmDFj2LVrV3MM3//+95k4cSJ79uzhe9/7Hjk5OYwePZqf//znzfusWbOGU045hezsbCZPnkxlZSWnnXYa69evb24zffp0NmzY0GnHOXRq+kB9dDoxDTvw1Dk9/Sg7kWtMh/y/VzaxOb+iU9/zpAHJ/Pz80W1uf/DBB9m4cWNzwnv33Xf55JNP2LhxY/NIlcWLF5Oenk5tbS2TJk3ikksuoVevQ2+Bum3bNp599ln++Mc/cvnll/Piiy9yzTXXHNLm1FNPZfXq1YgIf/rTn3jooYf43//9X+6//35SUlL44osvACgtLaWwsJCbbrqJ999/n6ysLEpK2r/j69atW3nyySeb/3J54IEHSE9Px+PxMHPmTDZs2MDIkSO54ooreO6555g0aRIVFRXExcVx44038tRTT/HrX/+ar776ivr6esaNGxf4gW5HQD19EZkrIltFJFdEFh6h3aUioiKS47dukW+/rSJydmcE3RaNSyfBU463wenpR1jSN6ZHmzx58iFDEx999FGys7OZOnUqe/bsYdu2bYftk5WVxfjxzj2xTz75ZHbt2nVYm7y8PM4++2zGjh3Lww8/zKZNmwBYsWIFt9xyS3O7tLQ0Vq9ezemnn94cR3p6ertxn3DCCUydOrV5+fnnn2fixIlMmDCBTZs2sXnzZrZu3Ur//v2ZNGkSAMnJyURGRnLZZZfx6quv0tjYyOLFi7n++uvbP1BHod2evoi4gMeA2UAesEZElqnq5hbtkoDbgY/91p0EzAdGAwOAFSIyQlWPXKDrIFdiBsklVXjqfD0Um4bBmA45Uo+8KyUkHOy4vfvuu6xYsYJVq1YRHx/PjBkzWh26GBMT0/za5XK1Wt657bbb+NGPfsS8efN49913uffeewGnBt9ydExr6wAiIyMPqdf7x+If986dO3nkkUdYs2YNaWlpXH/99dTV1bX5vvHx8cyePZulS5fy/PPPs3bt2tYOTYcF0tOfDOSq6g5VbQCWABe00u5+4CHA/1/hAmCJqtar6k4g1/d+x0V0ch8iRGkoyfOtsKRvTE+RlJREZWVlm9vLy8tJS0sjPj6eL7/8ktWrV3f4s8rLyxk40Ll96l/+8pfm9XPmzOG3v/1t83JpaSnTpk3jvffeY+fOnQDN5Z3MzEw+/fRTAD799NPm7S1VVFSQkJBASkoKBQUFvPbaawCMHDmS/Px81qxZA0BlZSVutxuAG2+8kdtvv51JkyYF9JfF0Qgk6Q8E9vgt5/nWNRORCcBgVX31aPf17b9ARNaKyNrCwsKAAm9NUnpfAFwVvo+0uXeM6TF69erF9OnTGTNmDHfeeedh2+fOnYvb7WbcuHH87Gc/O6R8crTuvfdeLrvsMk477TQyMjKa1999992UlpYyZswYsrOzWblyJb179+aJJ57g4osvJjs7myuuuAKASy65hJKSEsaPH8/jjz/OiBEjWv2s7OxsJkyYwOjRo/n2t7/N9OnTAYiOjua5557jtttuIzs7m9mzZzf/tXDyySeTnJzMDTfc0OGfsS2iqkduIHIZcLaq3uhb/hYwWVVv8y1HAO8A16vqLhF5F7hDVdeKyGPAKlX9m6/tn4HlqvpiW5+Xk5OjHf1zpvDz1+n90hUs80xjnmsV/PgrSOrbofcyJtxs2bKFUaNGBTsMA+Tn5zNjxgy+/PJLIiIO75u39m8lIutUNeewxi0E0tPPAwb7LQ8C8v2Wk4AxwLsisguYCizzncxtb99OlZbRH4CBUuSssJ6+MaaH+etf/8qUKVN44IEHWk34xyqQIZtrgOEikgXsxTkxe1XTRlUtB5r/PmrR068F/i4iv8Q5kTsc+KTzwj9UZFJvAAZIsbPCTuQaY3qYa6+9lmuvvfa4vX+7SV9V3SJyK/AG4AIWq+omEbkPWKuqy46w7yYReR7YDLiBW47XyB0A4pwTHn0ppZEoolwhdRmCMcYcs4CyoqouB5a3WHdPG21ntFh+AHigg/EdnahY6iLiiPXWUhsRS8dmmzbGmNAVMtMwNKmPcnr7jS6r5xtjTEshl/Q9cWkAuCNigxyJMcZ0PyGX9CMSnHPKHuvpG9OjHMvUygC//vWvqamp6cSIQlPIJf3oZGcEjyfSevrG9CShkPSbrqjtzkIu6cel9AEgKSklyJEYY45Gy6mVAR5++GEmTZrEuHHjmqckrq6u5rzzziM7O5sxY8bw3HPP8eijj5Kfn8+ZZ57JmWeeedh733fffUyaNIkxY8awYMECmi5Kzc3NZdasWWRnZzNx4kS2b98OwEMPPcTYsWPJzs5m4UJnjskZM2Y0z4NTVFREZmYmAE899RSXXXYZ559/PnPmzKGqqoqZM2cyceJExo4dy9KlS5vj+Otf/8q4cePIzs7mW9/6FpWVlWRlZdHY2Ag4UzZkZmY2Lx8PITemURKcaVaTkpKDHIkxPdhrC2H/F537nv3GwjkPtrm55dTKb775Jtu2beOTTz5BVZk3bx7vv/8+hYWFDBgwgH/961+AM49OSkoKv/zlL1m5cuUh0yo0ufXWW7nnHmfA4be+9S1effVVzj//fK6++moWLlzIRRddRF1dHV6vl9dee42XX36Zjz/+mPj4+ICmUl61ahUbNmwgPT0dt9vNSy+9RHJyMkVFRUydOpV58+axefNmHnjgAT788EMyMjIoKSkhKSmJGTNm8K9//YsLL7yQJUuWcMkllxAVdfzGHoZcT59439zadmGWMT3am2++yZtvvsmECROYOHEiX375Jdu2bWPs2LGsWLGCn/zkJ3zwwQekpLT/V/3KlSuZMmUKY8eO5Z133mHTpk1UVlayd+9eLrroIgBiY2OJj49nxYoV3HDDDcTHOzkkkAnPZs+e3dxOVfnpT3/KuHHjmDVrFnv37qWgoIB33nmHSy+9tPlLqan9jTfeyJNPPgnAk08+eVzm2/EXcj194n3f8jYFgzEdd4QeeVdRVRYtWsTNN9982LZ169axfPlyFi1axJw5c5p78a2pq6vj+9//PmvXrmXw4MHce++9zVMbt/W57U2l3HJKZ/+plJ955hkKCwtZt24dUVFRZGZmHnEq5enTp7Nr1y7ee+89PB4PY8aMafNn6Qyh29OPthuoGNOTtJxa+eyzz2bx4sVUVTl3wtu7dy8HDhwgPz+f+Ph4rrnmGu64447m6Y3bmpq5KUFnZGRQVVXFCy+8ADg3LRk0aBAvv/wyAPX19dTU1DBnzhwWL17cfFLYfyrldevWATS/R2vKy8vp06cPUVFRrFy5kq+//hqAmTNn8vzzz1NcXHzI+4Iz9cKVV1553Hv5EJI9/abyjvX0jelJ/KdWPuecc3j44YfZsmUL06ZNAyAxMZG//e1v5ObmcueddxIREUFUVBSPP/44AAsWLOCcc86hf//+rFy5svl9U1NTuemmmxg7diyZmZnNd6oCePrpp7n55pu55557iIqK4h//+Adz585l/fr15OTkEB0dzbnnnssvfvEL7rjjDi6//HKefvppzjrrrDZ/jquvvprzzz+fnJwcxo8fz8iRIwEYPXo0d911F2eccQYul4sJEybw1FNPNe9z9913c+WVV3b2YT1Mu1Mrd7VjmVoZgJoSeCgLzrobTj98Tm5jTOtsauXgeeGFF1i6dClPP/10QO2PZWrl0Ovpx6XBWT+Dky4MdiTGGNOu2267jddee43ly5e337gThF7SF4HT7wh2FMYYE5D/+7//69LPC70TucYYY9pkSd8Y06y7neMzhzvWfyNL+sYYwLk4qbi42BJ/N6aqFBcXExvb8bnFQq+mb4zpkEGDBpGXl0dhYWGwQzFHEBsby6BBgzq8f0BJX0TmAr/BuV3in1T1wRbbvwvcAniAKmCBqm4WkUxgC7DV13S1qn63w9EaY46bqKgosrKygh2GOc7aTfoi4gIeA2YDecAaEVmmqpv9mv1dVX/vaz8P+CUw17dtu6qO79ywjTHGdEQgNf3JQK6q7lDVBmAJcIF/A1Wt8FtMAKwoaIwx3VAgSX8gsMdvOc+37hAicouIbAceAm7325QlIp+JyHsictoxRWuMMeaYBFLTP3xauFZ68qr6GPCYiFwF3A1cB+wDhqhqsYicDLwsIqNb/GWAiCwAFvgWq0RkKx2XARQdw/5dobvH2N3jA4uxs1iMnaM7xHhCII0CSfp5wGC/5UFA/hHaLwEeB1DVeqDe93qd7y+BEcAhk+uo6hPAE4EE3B4RWRvI/BPB1N1j7O7xgcXYWSzGztETYmwSSHlnDTBcRLJEJBqYDyzzbyAiw/0WzwO2+db39p0IRkSGAsOBHZ0RuDHGmKPXbk9fVd0icivwBs6QzcWquklE7gPWquoy4FYRmQU0AqU4pR2A04H7RMSNM5zzu6ra/r3HjDHGHBcBjdNX1eXA8hbr7vF7/YM29nsRePFYAuyATikTHWfdPcbuHh9YjJ3FYuwcPSFGoBvOp2+MMeb4sbl3jDEmjIRM0heRuSKyVURyRWRhsOMBEJHBIrJSRLaIyCYR+YFvfbqIvCUi23zPad0gVpfveopXfctZIvKxL8bnfCfxgxlfqoi8ICJf+o7ntO50HEXkP3z/xhtF5FkRie0Ox1BEFovIARHZ6Leu1eMmjkd9v0MbRGRikOJ72PfvvEFEXhKRVL9ti3zxbRWRs493fG3F6LftDhFREcnwLXf5MTxaIZH0/aaKOAc4CbhSRE4KblQAuIEfq+ooYCpwiy+uhcDbqjoceNu3HGw/wJknqcn/AL/yxVgKfCcoUR30G+B1VR0JZOPE2i2Oo4gMxLkgMUdVx+AMeJhP9ziGT3FwSpQmbR23c3BG2A3HuW7m8SDF9xYwRlXHAV8BiwB8vzvzgdG+fX7XNDowCDEiIoNxpqfZ7bc6GMfw6Khqj38A04A3/JYXAYuCHVcrcS7F+U+yFejvW9cf2BrkuAbh/PKfBbyKc0FeERDZ2vENQnzJwE5856D81neL48jBq9bTcQZHvAqc3V2OIZAJbGzvuAF/AK5srV1Xxtdi20XAM77Xh/xe44wonBaMY+hb9wJOB2QXkBHMY3g0j5Do6RPgVBHBJM6MoxOAj4G+qroPwPfcJ3iRAfBr4D8Br2+5F1Cmqm7fcrCP51CgEHjSV4L6k4gk0E2Oo6ruBR7B6fHtA8qBdXSvY+ivrePWHX+Pvg285nvdbeITZ2LJvar6eYtN3SbGtoRK0g9oqohgEZFEnKGrP9QWU1AEm4h8Ezigquv8V7fSNJjHMxKYCDyuqhOAarpHSQwAX038AiALGIAz6eA5rTTtNv8n29Ct/t1F5C6cEukzTataadbl8YlIPHAXcE9rm1tZ163+3UMl6R/tVBFdRkSicBL+M6r6T9/qAhHp79veHzgQrPiA6cA8EdmFM4XGWTg9/1QRabqOI9jHMw/IU9WPfcsv4HwJdJfjOAvYqaqFqtoI/BM4he51DP21ddy6ze+RiFwHfBO4Wn11ErpPfMNwvuA/9/3eDAI+FZF+dJ8Y2xQqSb/dqSKCQUQE+DOwRVV/6bdpGQevWr4Op9YfFKq6SFUHqWomznF7R1WvBlYCl/qaBTvG/cAeETnRt2omsJnucxx3A1NFJN73b94UX7c5hi20ddyWAdf6RqBMBcqbykBdSZybNv0EmKeqNX6blgHzRSRGRLJwTpZ+0tXxqeoXqtpHVTN9vzd5wETf/9NucQyPKNgnFTrxRMu5OGf6twN3BTseX0yn4vxptwFY73uci1MzfxtnjqK3gfRgx+qLdwbwqu/1UJxfqFzgH0BMkGMbjzNR3wbgZSCtOx1H4P8BXwIbgaeBmO5wDIFncc4zNOIkp++0ddxwShOP+X6HvsAZjRSM+HJx6uJNvzO/92t/ly++rcA5wTqGLbbv4uCJ3C4/hkf7sCtyjTEmjIRKeccYY0wALOkbY0wYsaRvjDFhxJK+McaEEUv6xhgTRizpG2NMGLGkb4wxYcSSvjHGhJH/D43inG42LPH1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_accuracies, label = \"train accuracy\")\n",
    "plt.plot(test_accuracies, label = \"test accuracy\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(cnn_net.state_dict(), \"data/cnn_gaussian_6layer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_pytorch_p36)",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}