DL_Course_SamU/lab_4/assignment4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Лабораторная работа 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tensorflow 2.x\n",
    "\n",
    "1) Подготовка данных\n",
    "\n",
    "2) Использование Keras Model API\n",
    "\n",
    "3) Использование Keras Sequential + Functional API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.tensorflow.org/tutorials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для выполнения лабораторной работы необходимо установить tensorflow версии 2.0 или выше .\n",
    "\n",
    "Рекомендуется использовать возможности Colab'а по обучению моделей на GPU.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Подготовка данных\n",
    "Загрузите набор данных из предыдущей лабораторной работы. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cifar10(num_training=49000, num_validation=1000, num_test=10000):\n",
    "    \"\"\"\n",
    "    Fetch the CIFAR-10 dataset from the web and perform preprocessing to prepare\n",
    "    it for the two-layer neural net classifier. These are the same steps as\n",
    "    we used for the SVM, but condensed to a single function.\n",
    "    \"\"\"\n",
    "    # Load the raw CIFAR-10 dataset and use appropriate data types and shapes\n",
    "    cifar10 = tf.keras.datasets.cifar10.load_data()\n",
    "    (X_train, y_train), (X_test, y_test) = cifar10\n",
    "    X_train = np.asarray(X_train, dtype=np.float32)\n",
    "    y_train = np.asarray(y_train, dtype=np.int32).flatten()\n",
    "    X_test = np.asarray(X_test, dtype=np.float32)\n",
    "    y_test = np.asarray(y_test, dtype=np.int32).flatten()\n",
    "\n",
    "    # Subsample the data\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = X_train[mask]\n",
    "    y_val = y_train[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = X_train[mask]\n",
    "    y_train = y_train[mask]\n",
    "    mask = range(num_test)\n",
    "    X_test = X_test[mask]\n",
    "    y_test = y_test[mask]\n",
    "\n",
    "    # Normalize the data: subtract the mean pixel and divide by std\n",
    "    mean_pixel = X_train.mean(axis=(0, 1, 2), keepdims=True)\n",
    "    std_pixel = X_train.std(axis=(0, 1, 2), keepdims=True)\n",
    "    X_train = (X_train - mean_pixel) / std_pixel\n",
    "    X_val = (X_val - mean_pixel) / std_pixel\n",
    "    X_test = (X_test - mean_pixel) / std_pixel\n",
    "\n",
    "    return X_train, y_train, X_val, y_val, X_test, y_test\n",
    "\n",
    "# If there are errors with SSL downloading involving self-signed certificates,\n",
    "# it may be that your Python version was recently installed on the current machine.\n",
    "# See: https://github.com/tensorflow/tensorflow/issues/10779\n",
    "# To fix, run the command: /Applications/Python\\ 3.7/Install\\ Certificates.command\n",
    "#   ...replacing paths as necessary.\n",
    "\n",
    "# Invoke the above function to get our data.\n",
    "NHW = (0, 1, 2)\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_cifar10()\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape, y_train.dtype)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(object):\n",
    "    def __init__(self, X, y, batch_size, shuffle=False):\n",
    "        \"\"\"\n",
    "        Construct a Dataset object to iterate over data X and labels y\n",
    "        \n",
    "        Inputs:\n",
    "        - X: Numpy array of data, of any shape\n",
    "        - y: Numpy array of labels, of any shape but with y.shape[0] == X.shape[0]\n",
    "        - batch_size: Integer giving number of elements per minibatch\n",
    "        - shuffle: (optional) Boolean, whether to shuffle the data on each epoch\n",
    "        \"\"\"\n",
    "        assert X.shape[0] == y.shape[0], 'Got different numbers of data and labels'\n",
    "        self.X, self.y = X, y\n",
    "        self.batch_size, self.shuffle = batch_size, shuffle\n",
    "\n",
    "    def __iter__(self):\n",
    "        N, B = self.X.shape[0], self.batch_size\n",
    "        idxs = np.arange(N)\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(idxs)\n",
    "        return iter((self.X[i:i+B], self.y[i:i+B]) for i in range(0, N, B))\n",
    "\n",
    "\n",
    "train_dset = Dataset(X_train, y_train, batch_size=64, shuffle=True)\n",
    "val_dset = Dataset(X_val, y_val, batch_size=64, shuffle=False)\n",
    "test_dset = Dataset(X_test, y_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can iterate through a dataset like this:\n",
    "for t, (x, y) in enumerate(train_dset):\n",
    "    print(t, x.shape, y.shape)\n",
    "    if t > 5: break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Keras Model Subclassing API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Для реализации собственной модели с помощью Keras Model Subclassing API необходимо выполнить следующие шаги:\n",
    "\n",
    "1) Определить новый класс, который является наследником tf.keras.Model.\n",
    "\n",
    "2) В методе __init__() определить все необходимые слои из модуля tf.keras.layer\n",
    "\n",
    "3) Реализовать прямой проход в методе call() на основе слоев, объявленных в __init__()\n",
    "\n",
    "Ниже приведен пример использования keras API для определения двухслойной полносвязной сети. \n",
    "\n",
    "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TwoLayerFC(tf.keras.Model):\n",
    "    def __init__(self, hidden_size, num_classes):\n",
    "        super(TwoLayerFC, self).__init__()        \n",
    "        initializer = tf.initializers.VarianceScaling(scale=2.0)\n",
    "        self.fc1 = tf.keras.layers.Dense(hidden_size, activation='relu',\n",
    "                                   kernel_initializer=initializer)\n",
    "        self.fc2 = tf.keras.layers.Dense(num_classes, activation='softmax',\n",
    "                                   kernel_initializer=initializer)\n",
    "        self.flatten = tf.keras.layers.Flatten()\n",
    "    \n",
    "    def call(self, x, training=False):\n",
    "        x = self.flatten(x)\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "def test_TwoLayerFC():\n",
    "    \"\"\" A small unit test to exercise the TwoLayerFC model above. \"\"\"\n",
    "    input_size, hidden_size, num_classes = 50, 42, 10\n",
    "    x = tf.zeros((64, input_size))\n",
    "    model = TwoLayerFC(hidden_size, num_classes)\n",
    "    with tf.device(device):\n",
    "        scores = model(x)\n",
    "        print(scores.shape)\n",
    "        \n",
    "test_TwoLayerFC()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Реализуйте трехслойную CNN для вашей задачи классификации. \n",
    "\n",
    "Архитектура сети:\n",
    "    \n",
    "1. Сверточный слой (5 x 5 kernels, zero-padding = 2)\n",
    "2. Функция активации ReLU \n",
    "3. Сверточный слой (3 x 3 kernels, zero-padding = 1)\n",
    "4. Функция активации ReLU \n",
    "5. Полносвязный слой \n",
    "6. Функция активации Softmax \n",
    "\n",
    "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/Conv2D\n",
    "\n",
    "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ThreeLayerConvNet(tf.keras.Model):\n",
    "    def __init__(self, channel_1, channel_2, num_classes):\n",
    "        super(ThreeLayerConvNet, self).__init__()\n",
    "        ########################################################################\n",
    "        # TODO: Implement the __init__ method for a three-layer ConvNet. You   #\n",
    "        # should instantiate layer objects to be used in the forward pass.     #\n",
    "        ########################################################################\n",
    "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "        pass\n",
    "\n",
    "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "        ########################################################################\n",
    "        #                           END OF YOUR CODE                           #\n",
    "        ########################################################################\n",
    "        \n",
    "    def call(self, x, training=False):\n",
    "        scores = None\n",
    "        ########################################################################\n",
    "        # TODO: Implement the forward pass for a three-layer ConvNet. You      #\n",
    "        # should use the layer objects defined in the __init__ method.         #\n",
    "        ########################################################################\n",
    "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "        pass\n",
    "\n",
    "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "        ########################################################################\n",
    "        #                           END OF YOUR CODE                           #\n",
    "        ########################################################################        \n",
    "        return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_ThreeLayerConvNet():    \n",
    "    channel_1, channel_2, num_classes = 12, 8, 10\n",
    "    model = ThreeLayerConvNet(channel_1, channel_2, num_classes)\n",
    "    with tf.device(device):\n",
    "        x = tf.zeros((64, 3, 32, 32))\n",
    "        scores = model(x)\n",
    "        print(scores.shape)\n",
    "\n",
    "test_ThreeLayerConvNet()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пример реализации процесса обучения:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_part34(model_init_fn, optimizer_init_fn, num_epochs=1, is_training=False):\n",
    "    \"\"\"\n",
    "    Simple training loop for use with models defined using tf.keras. It trains\n",
    "    a model for one epoch on the CIFAR-10 training set and periodically checks\n",
    "    accuracy on the CIFAR-10 validation set.\n",
    "    \n",
    "    Inputs:\n",
    "    - model_init_fn: A function that takes no parameters; when called it\n",
    "      constructs the model we want to train: model = model_init_fn()\n",
    "    - optimizer_init_fn: A function which takes no parameters; when called it\n",
    "      constructs the Optimizer object we will use to optimize the model:\n",
    "      optimizer = optimizer_init_fn()\n",
    "    - num_epochs: The number of epochs to train for\n",
    "    \n",
    "    Returns: Nothing, but prints progress during trainingn\n",
    "    \"\"\"    \n",
    "    with tf.device(device):\n",
    "\n",
    "        \n",
    "        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "        \n",
    "        model = model_init_fn()\n",
    "        optimizer = optimizer_init_fn()\n",
    "        \n",
    "        train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "        train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
    "    \n",
    "        val_loss = tf.keras.metrics.Mean(name='val_loss')\n",
    "        val_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='val_accuracy')\n",
    "        \n",
    "        t = 0\n",
    "        for epoch in range(num_epochs):\n",
    "            \n",
    "            # Reset the metrics - https://www.tensorflow.org/alpha/guide/migration_guide#new-style_metrics\n",
    "            train_loss.reset_states()\n",
    "            train_accuracy.reset_states()\n",
    "            \n",
    "            for x_np, y_np in train_dset:\n",
    "                with tf.GradientTape() as tape:\n",
    "                    \n",
    "                    # Use the model function to build the forward pass.\n",
    "                    scores = model(x_np, training=is_training)\n",
    "                    loss = loss_fn(y_np, scores)\n",
    "      \n",
    "                    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                    \n",
    "                    # Update the metrics\n",
    "                    train_loss.update_state(loss)\n",
    "                    train_accuracy.update_state(y_np, scores)\n",
    "                    \n",
    "                    if t % print_every == 0:\n",
    "                        val_loss.reset_states()\n",
    "                        val_accuracy.reset_states()\n",
    "                        for test_x, test_y in val_dset:\n",
    "                            # During validation at end of epoch, training set to False\n",
    "                            prediction = model(test_x, training=False)\n",
    "                            t_loss = loss_fn(test_y, prediction)\n",
    "\n",
    "                            val_loss.update_state(t_loss)\n",
    "                            val_accuracy.update_state(test_y, prediction)\n",
    "                        \n",
    "                        template = 'Iteration {}, Epoch {}, Loss: {}, Accuracy: {}, Val Loss: {}, Val Accuracy: {}'\n",
    "                        print (template.format(t, epoch+1,\n",
    "                                             train_loss.result(),\n",
    "                                             train_accuracy.result()*100,\n",
    "                                             val_loss.result(),\n",
    "                                             val_accuracy.result()*100))\n",
    "                    t += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size, num_classes = 4000, 10\n",
    "learning_rate = 1e-2\n",
    "\n",
    "def model_init_fn():\n",
    "    return TwoLayerFC(hidden_size, num_classes)\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    return tf.keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Обучите трехслойную CNN. В tf.keras.optimizers.SGD укажите Nesterov momentum = 0.9 . \n",
    "\n",
    "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/optimizers/SGD\n",
    "\n",
    "Значение accuracy на валидационной выборке после 1 эпохи обучения должно быть > 0.5 ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 3e-3\n",
    "channel_1, channel_2, num_classes = 32, 16, 10\n",
    "\n",
    "def model_init_fn():\n",
    "    model = None\n",
    "    ############################################################################\n",
    "    # TODO: Complete the implementation of model_fn.                           #\n",
    "    ############################################################################\n",
    "    # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "    pass\n",
    "\n",
    "    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "    ############################################################################\n",
    "    #                           END OF YOUR CODE                               #\n",
    "    ############################################################################\n",
    "    return model\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    optimizer = None\n",
    "    ############################################################################\n",
    "    # TODO: Complete the implementation of model_fn.                           #\n",
    "    ############################################################################\n",
    "    # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "    pass\n",
    "\n",
    "    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "    ############################################################################\n",
    "    #                           END OF YOUR CODE                               #\n",
    "    ############################################################################\n",
    "    return optimizer\n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Использование Keras Sequential API для реализации последовательных моделей.\n",
    "\n",
    "Пример для полносвязной сети:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-2\n",
    "\n",
    "def model_init_fn():\n",
    "    input_shape = (32, 32, 3)\n",
    "    hidden_layer_size, num_classes = 4000, 10\n",
    "    initializer = tf.initializers.VarianceScaling(scale=2.0)\n",
    "    layers = [\n",
    "        tf.keras.layers.Flatten(input_shape=input_shape),\n",
    "        tf.keras.layers.Dense(hidden_layer_size, activation='relu',\n",
    "                              kernel_initializer=initializer),\n",
    "        tf.keras.layers.Dense(num_classes, activation='softmax', \n",
    "                              kernel_initializer=initializer),\n",
    "    ]\n",
    "    model = tf.keras.Sequential(layers)\n",
    "    return model\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    return tf.keras.optimizers.SGD(learning_rate=learning_rate) \n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Альтернативный менее гибкий способ обучения:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model_init_fn()\n",
    "model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate),\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=[tf.keras.metrics.sparse_categorical_accuracy])\n",
    "model.fit(X_train, y_train, batch_size=64, epochs=1, validation_data=(X_val, y_val))\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Перепишите реализацию трехслойной CNN с помощью tf.keras.Sequential API . Обучите модель двумя способами."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_init_fn():\n",
    "    model = None\n",
    "    ############################################################################\n",
    "    # TODO: Construct a three-layer ConvNet using tf.keras.Sequential.         #\n",
    "    ############################################################################\n",
    "    # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "    pass\n",
    "\n",
    "    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "    ############################################################################\n",
    "    #                            END OF YOUR CODE                              #\n",
    "    ############################################################################\n",
    "    return model\n",
    "\n",
    "learning_rate = 5e-4\n",
    "def optimizer_init_fn():\n",
    "    optimizer = None\n",
    "    ############################################################################\n",
    "    # TODO: Complete the implementation of model_fn.                           #\n",
    "    ############################################################################\n",
    "    # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "    pass\n",
    "\n",
    "    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "    ############################################################################\n",
    "    #                           END OF YOUR CODE                               #\n",
    "    ############################################################################\n",
    "    return optimizer\n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model_init_fn()\n",
    "model.compile(optimizer='sgd',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=[tf.keras.metrics.sparse_categorical_accuracy])\n",
    "model.fit(X_train, y_train, batch_size=64, epochs=1, validation_data=(X_val, y_val))\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Использование Keras Functional API\n",
    "\n",
    "Для реализации более сложных архитектур сети с несколькими входами/выходами, повторным использованием слоев, \"остаточными\" связями (residual connections) необходимо явно указать входные и выходные тензоры. \n",
    "\n",
    "Ниже представлен пример для полносвязной сети. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def two_layer_fc_functional(input_shape, hidden_size, num_classes):  \n",
    "    initializer = tf.initializers.VarianceScaling(scale=2.0)\n",
    "    inputs = tf.keras.Input(shape=input_shape)\n",
    "    flattened_inputs = tf.keras.layers.Flatten()(inputs)\n",
    "    fc1_output = tf.keras.layers.Dense(hidden_size, activation='relu',\n",
    "                                 kernel_initializer=initializer)(flattened_inputs)\n",
    "    scores = tf.keras.layers.Dense(num_classes, activation='softmax',\n",
    "                             kernel_initializer=initializer)(fc1_output)\n",
    "\n",
    "    # Instantiate the model given inputs and outputs.\n",
    "    model = tf.keras.Model(inputs=inputs, outputs=scores)\n",
    "    return model\n",
    "\n",
    "def test_two_layer_fc_functional():\n",
    "    \"\"\" A small unit test to exercise the TwoLayerFC model above. \"\"\"\n",
    "    input_size, hidden_size, num_classes = 50, 42, 10\n",
    "    input_shape = (50,)\n",
    "    \n",
    "    x = tf.zeros((64, input_size))\n",
    "    model = two_layer_fc_functional(input_shape, hidden_size, num_classes)\n",
    "    \n",
    "    with tf.device(device):\n",
    "        scores = model(x)\n",
    "        print(scores.shape)\n",
    "        \n",
    "test_two_layer_fc_functional()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = (32, 32, 3)\n",
    "hidden_size, num_classes = 4000, 10\n",
    "learning_rate = 1e-2\n",
    "\n",
    "def model_init_fn():\n",
    "    return two_layer_fc_functional(input_shape, hidden_size, num_classes)\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    return tf.keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Filter size**: Above we used 5x5 and 3x3; is this optimal?\n",
    "- **Number of filters**: Above we used 16 and 32 filters. Would more or fewer do better?\n",
    "- **Pooling**: We didn't use any pooling above. Would this improve the model?\n",
    "- **Normalization**: Would your model be improved with batch normalization, layer normalization, group normalization, or some other normalization strategy?\n",
    "- **Network architecture**: The ConvNet above has only three layers of trainable parameters. Would a deeper model do better?\n",
    "- **Global average pooling**: Instead of flattening after the final convolutional layer, would global average pooling do better? This strategy is used for example in Google's Inception network and in Residual Networks.\n",
    "- **Regularization**: Would some kind of regularization improve performance? Maybe weight decay or dropout?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Поэкспериментируйте с архитектурой сверточной сети. Для вашего набора данных вам необходимо получить как минимум 70% accuracy на валидационной выборке за 10 эпох обучения. Опишите все эксперименты и сделайте выводы (без выполнения данного пункта работы приниматься не будут). \n",
    "\n",
    "Эспериментируйте с архитектурой, гиперпараметрами, функцией потерь, регуляризацией, методом оптимизации.  \n",
    "\n",
    "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/BatchNormalization#methods https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/Dropout#methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomConvNet(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(CustomConvNet, self).__init__()\n",
    "        ############################################################################\n",
    "        # TODO: Construct a model that performs well on CIFAR-10                   #\n",
    "        ############################################################################\n",
    "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "        pass\n",
    "\n",
    "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "        ############################################################################\n",
    "        #                            END OF YOUR CODE                              #\n",
    "        ############################################################################\n",
    "    \n",
    "    def call(self, input_tensor, training=False):\n",
    "        ############################################################################\n",
    "        # TODO: Construct a model that performs well on CIFAR-10                   #\n",
    "        ############################################################################\n",
    "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "\n",
    "        pass\n",
    "\n",
    "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
    "        ############################################################################\n",
    "        #                            END OF YOUR CODE                              #\n",
    "        ############################################################################\n",
    "        return x\n",
    "\n",
    "\n",
    "print_every = 700\n",
    "num_epochs = 10\n",
    "\n",
    "model = CustomConvNet()\n",
    "\n",
    "def model_init_fn():\n",
    "    return CustomConvNet()\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    learning_rate = 1e-3\n",
    "    return tf.keras.optimizers.Adam(learning_rate) \n",
    "\n",
    "train_part34(model_init_fn, optimizer_init_fn, num_epochs=num_epochs, is_training=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Опишите все эксперименты, результаты. Сделайте выводы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}