Adversarial Examples for Traffic Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "\n",
    "- Use model trained by V2Ray server in AWS to classify data from V2Ray server in GCE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# v2ray traffic tag\n",
    "V2RAY_HOST_TAG =  # For privacy concern, please provide \"Your-V2Ray-Server-IP-and-Port\"\n",
    "# modified packet data from Google Compute Engine\n",
    "DATA_ROOT = '/src/data/tcpsorter/mod-tcpsorter.gce'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from pathlib import Path\n",
    "from tensorflow.keras.utils import Sequence\n",
    "\n",
    "# packet data size\n",
    "FIXED_PACKET_SIZE = 1500\n",
    "NUM_OF_PACKETS_PER_FILE = 16\n",
    "# rescale factor for packet data byte\n",
    "RESCALE_FACTOR = np.float32(1./255)\n",
    "# training data vs validation data\n",
    "TRAINING_DATA_PERCENTAGE = 0.8\n",
    "# modified packet file extension\n",
    "PACKET_FILE_EXT = '*.bin'\n",
    "\n",
    "\n",
    "def rglob(data_root, file_ext):\n",
    "    files = list()\n",
    "    for filePath in Path(data_root).rglob(file_ext):\n",
    "        files.append(str(filePath))\n",
    "    return files\n",
    "\n",
    "\n",
    "def binary_classification(packet_path, match_string=V2RAY_HOST_TAG):\n",
    "    \"\"\"Binary network traffic classification function\n",
    "\n",
    "    :param packet_path: file path to packet\n",
    "    :param match_string:\n",
    "    :return: 1, if it is v2ray traffice. 0, otherwise.\n",
    "    \"\"\"\n",
    "    if packet_path.find(match_string) != -1:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "\n",
    "def generate_v2ray_packet_path_list(data_root=DATA_ROOT, is_v2ray=True):\n",
    "    file_list = rglob(data_root, PACKET_FILE_EXT)\n",
    "    if is_v2ray:\n",
    "        bool_value_class = 1\n",
    "        string_format = 'Total v2ray traffic %d'\n",
    "    else:\n",
    "        bool_value_class = 0\n",
    "        string_format = 'Total non v2ray traffic %d'\n",
    "    file_cat_list = [file_path for file_path in file_list if binary_classification(file_path) == bool_value_class]\n",
    "    file_cat_list.sort()\n",
    "    print(string_format % len(file_cat_list))\n",
    "    return file_cat_list\n",
    "\n",
    "\n",
    "def generate_train_validation_packet_path_list(data_root=DATA_ROOT, training_pct=TRAINING_DATA_PERCENTAGE, eqaul_size=True):\n",
    "    file_list = rglob(data_root, PACKET_FILE_EXT)\n",
    "    v2ray_file_list = [file_path for file_path in file_list if binary_classification(file_path) == 1]\n",
    "    non_v2ray_file_list = [file_path for file_path in file_list if binary_classification(file_path) == 0]\n",
    "    v2ray_file_list.sort()\n",
    "    non_v2ray_file_list.sort()\n",
    "\n",
    "    if eqaul_size:\n",
    "        cut_off_count = min(len(v2ray_file_list), len(non_v2ray_file_list))\n",
    "        v2ray_file_size = cut_off_count\n",
    "        non_v2ray_file_size = cut_off_count\n",
    "    else:\n",
    "        v2ray_file_size = len(v2ray_file_list)\n",
    "        non_v2ray_file_size = len(non_v2ray_file_list)\n",
    "\n",
    "    v2ray_indexes = np.arange(len(v2ray_file_list))\n",
    "    np.random.shuffle(v2ray_indexes)\n",
    "    non_v2ray_indexes = np.arange(len(non_v2ray_file_list))\n",
    "    np.random.shuffle(non_v2ray_indexes)\n",
    "\n",
    "    training_file_list = [v2ray_file_list[index]\n",
    "                          for index in v2ray_indexes[:math.ceil(v2ray_file_size * training_pct)]] + \\\n",
    "                         [non_v2ray_file_list[index]\n",
    "                          for index in non_v2ray_indexes[:math.ceil(non_v2ray_file_size * training_pct)]]\n",
    "\n",
    "    validation_file_list = [v2ray_file_list[index]\n",
    "                            for index in v2ray_indexes[math.ceil(v2ray_file_size * training_pct): v2ray_file_size]] + \\\n",
    "                           [non_v2ray_file_list[index]\n",
    "                            for index in non_v2ray_indexes[math.ceil(non_v2ray_file_size * training_pct): non_v2ray_file_size]]\n",
    "\n",
    "    print(\"Statistics: \")\n",
    "    print(\"Total V2ray traffic %d, Total non-V2ray traffic %d\" % (len(v2ray_file_list), len(non_v2ray_file_list)))\n",
    "    print(\"Output train traffic %d, Total validation traffic %d\" % (len(training_file_list), len(validation_file_list)))\n",
    "\n",
    "    return training_file_list, validation_file_list\n",
    "\n",
    "\n",
    "class PacketDataGenerator(Sequence):\n",
    "    \"\"\"Generates data for Keras\n",
    "    Sequence based data generator. Suitable for building data generator for training and prediction.\n",
    "    \"\"\"\n",
    "    def __init__(self, data_path_list, classification_func=binary_classification, rescale=RESCALE_FACTOR,\n",
    "                 batch_size=32, dim=(NUM_OF_PACKETS_PER_FILE, FIXED_PACKET_SIZE), shuffle=True, predict=False):\n",
    "        \"\"\"Initialization\n",
    "\n",
    "        :param data_path_list: list of data path\n",
    "        :param classification_func: binary classification function\n",
    "        :param batch_size: batch size at each iteration\n",
    "        :param dim: tuple indicating data dimension\n",
    "        :param shuffle: True to shuffle label indexes after every epoch\n",
    "        \"\"\"\n",
    "        self.data_path_list = data_path_list\n",
    "        self.classification_func = classification_func\n",
    "        self.rescale = rescale\n",
    "        self.batch_size = batch_size\n",
    "        self.dim = dim\n",
    "        self.shuffle = shuffle\n",
    "        self.predict = predict\n",
    "        self.indexes = np.arange(len(self.data_path_list))\n",
    "        self.on_epoch_end()\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Denotes the number samples of batches per epoch\n",
    "\n",
    "        :return: number of batches per epoch\n",
    "        \"\"\"\n",
    "        return int(np.floor(len(self.data_path_list) / self.batch_size))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"Generate one batch of data\n",
    "\n",
    "        :param index: index of the batch\n",
    "        :return: X and y when fitting. X only when predicting\n",
    "        \"\"\"\n",
    "        # Generate indexes of the batch\n",
    "        indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "\n",
    "        # Generate sub packet path list\n",
    "        data_path_sub_list = [self.data_path_list[k] for k in indexes]\n",
    "\n",
    "        # Generate data\n",
    "        x = self.standardize(self._generate_x(data_path_sub_list))\n",
    "        if self.predict:\n",
    "            return x\n",
    "        else:\n",
    "            y = self._generate_y(data_path_sub_list)\n",
    "            return x, y\n",
    "\n",
    "    def standardize(self, x):\n",
    "        if self.rescale:\n",
    "            x *= self.rescale\n",
    "        return x\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        \"\"\"Updates indexes after each epoch\n",
    "\n",
    "        \"\"\"\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(self.indexes)\n",
    "\n",
    "    def _generate_x(self, file_path_list):\n",
    "        \"\"\"Generates data containing batch_size data\n",
    "\n",
    "        :param file_path_list: file path list\n",
    "        :return: batch of data\n",
    "        \"\"\"\n",
    "        # Initialization\n",
    "        x = np.empty((self.batch_size, *self.dim), dtype=np.float32)\n",
    "\n",
    "        # Generate data\n",
    "        for i, file_path in enumerate(file_path_list):\n",
    "            # Store sample\n",
    "            x[i, ] = self._load_packet(file_path)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def _generate_y(self, file_path_list):\n",
    "        \"\"\"Generates data containing batch_size\n",
    "\n",
    "        :param file_path_list: file path list\n",
    "        :return: batch if masks\n",
    "        \"\"\"\n",
    "        y = np.empty((self.batch_size, ), dtype=np.float32)\n",
    "\n",
    "        # Generate data\n",
    "        for i, file_path in enumerate(file_path_list):\n",
    "            y[i, ] = self.classification_func(file_path)\n",
    "\n",
    "        return y\n",
    "\n",
    "    def _load_packet(self, packet_path):\n",
    "        \"\"\"Load packet\n",
    "\n",
    "        :param packet_path: path to packet\n",
    "        :return: packet in numpy array\n",
    "        \"\"\"\n",
    "        packet_dt = np.dtype((np.uint8, FIXED_PACKET_SIZE))\n",
    "        packet_data = np.fromfile(packet_path, dtype=packet_dt, count=NUM_OF_PACKETS_PER_FILE)\n",
    "        return packet_data.astype(np.float32)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Previously Trained Model\n",
    "\n",
    "- Classify the traffic data from Google Compute Engine by model trained by the traffic data from AWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "model = load_model('/src/data/tcpsorter/model/v2ray_classifier_equal_sample.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total v2ray traffic 780\n",
      "Total non v2ray traffic 3241\n"
     ]
    }
   ],
   "source": [
    "# Create tes data generator\n",
    "v2ray_packet_file_list = generate_v2ray_packet_path_list()\n",
    "non_v2ray_packet_file_list = generate_v2ray_packet_path_list(is_v2ray=False)\n",
    "v2ray_generator = PacketDataGenerator(v2ray_packet_file_list, shuffle=False)\n",
    "non_v2ray_generator = PacketDataGenerator(non_v2ray_packet_file_list, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Adversarial Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FGSM Approach\n",
    "\n",
    "- Fast Gradient Sign Method. See [paper](https://arxiv.org/abs/1412.6572)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keras version: 2.3.1\n",
      "tensorflow version: 2.0.0\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "print('keras version: %s' % keras.__version__)\n",
    "print('tensorflow version: %s' % tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = v2ray_generator.__getitem__(0)\n",
    "len(v2ray_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_detail(var):\n",
    "    if var.shape is not None:\n",
    "        print('shape: ', var.shape)\n",
    "    if var.dtype is not None:\n",
    "        print('dtype: ', var.dtype)\n",
    "    print('type: ', type(var))\n",
    "    print('content: ', var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (32, 16, 1500)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]]]\n",
      "shape:  (32,)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n",
      " 1. 1. 1. 1. 1. 1. 1. 1.]\n"
     ]
    }
   ],
   "source": [
    "# 32 samples in each batch, 16 packets in each connection, 1500 byte in each packet\n",
    "print_detail(x)\n",
    "print_detail(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick the first input as an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (1, 16, 1500)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]]\n"
     ]
    }
   ],
   "source": [
    "# pick first example\n",
    "x_in = x[0:1, ]\n",
    "print_detail(x_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (1, 1)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[1.]]\n"
     ]
    }
   ],
   "source": [
    "y_pre = model.predict(x_in)\n",
    "print_detail(y_pre)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ricky's Comment\n",
    "\n",
    "- The prediction from trained model is 1, which matches the true V2Ray traffic label. \n",
    "- Now we want to modify input x so that the prediction is 0. In other word, the probability of V2Ray traffic is 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute adversaril sign gradient\n",
    "@tf.function\n",
    "def adversarial_sign_gradient(x, y_target, trained_model=model):\n",
    "    x = tf.cast(x, tf.float32)\n",
    "    with tf.GradientTape() as tape:\n",
    "        tape.watch(x)\n",
    "        y_pred = trained_model(x)\n",
    "        loss_value = tf.keras.losses.binary_crossentropy(y_target, y_pred)\n",
    "    \n",
    "    gradient = tape.gradient(loss_value, x)\n",
    "    signed_grad = tf.cast(tf.math.sign(gradient), dtype=tf.int32)\n",
    "    return signed_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "adv_sign = adversarial_sign_gradient(x_in, np.array([[0.]])).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:  (1, 16, 1500)\n",
      "dtype:  int32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[-1 -1 -1 ... -1 -1 -1]\n",
      "  [-1 -1 -1 ...  1  1  1]\n",
      "  [-1 -1 -1 ... -1 -1 -1]\n",
      "  ...\n",
      "  [-1 -1 -1 ... -1  1  1]\n",
      "  [-1 -1 -1 ... -1  1 -1]\n",
      "  [ 0  0  0 ...  0  0  0]]]\n"
     ]
    }
   ],
   "source": [
    "print_detail(adv_sign)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAAmCAYAAADducBhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAIqElEQVR4nO2da6hdRxXHf38Tk9ooTWK0TZvoTaRUgqh5gAlKKa1NYygNQj8kFky1UlAEXyCJQaGg0IeID8RUfKASYzWtGgIStNZvEpOrJk3TXnNrY5vQmEawvhBaXH6YOTeT7ey9z7m9dz9k/eBwZs/MnvnPOmuvPWf2PvvIzHAcx3H6x8vaFuA4juNMDw/gjuM4PcUDuOM4Tk/xAO44jtNTPIA7juP0FA/gjuM4PeUlBXBJmyRNSJqUtGOmRDmO4zj1aLr3gUuaA/wBuBE4DRwGtpnZiZmT5ziO45RROwOX9C1J5yQdT/IWA78GrgDuBxYAPwC2zJZQx3Ec52JqZ+CSrgX+AXzXzN4U8+4FrgT+DUwCi4DjwNvM7MOF/e8E7gRgAWt5Y8hfOw7ja/N9DsrWjleLT/evq5vbt6ih2EZRX65+XZ0iZeXFMQ/TT1ovrVOnMx3nqFrTfnPpMnJ1Rm1jGM3/79T51yjtwEtra6a0zBRVx+Mwx1BVe8PEq1H0DZue0jfOeTN7TbHNoZZQJI0BB5IAPgHcB6wHPg38CvgsmQB+UTvrZBwJaRMo03Wab0qF5uuW9mUX2qqqV0XaxjCMUje3X1Fv0Q51dXLpXB/p+Op0Ffst057bp05LVT9l48y1kfOZYllx3MPYPNfGMO/DjrOopU5/nT/W9V9lj7L6qaa0bq68yp+qbDTYd7q+Xddnsf+0v+K4hvX5ujpVWnO+WNz+n/GKcTNbV2xruhcxLwceA5YDZ+P2MuBM3Y6p2FxgLXPkNM9UH5SL5YN20vZk+Vduv2L/ufei9oHOsjEPynLaiu3WHZg5Hbm8OocqOlO6X9GmZZ9fVWDNvdf1UxxPcfy5E37adrGPYt9VAajq4Mrl59pOfaA4vqr2irbKaSqzSbFu2RhzenJ9pPtWHX/F8eY+yzIfK/ZTpj93fOXGUDeJKOurakx1vpJro+6zHPXzT5lbXlTLYeBqYAwwYCvwnmKli5ZQXlcuMs1LKTswquoUy8razznFMIGxbkY0jJYqzcU2qmYKOepsWrVP2QygbJzF8RTTue26GWjqG8W6VQEw97mUBemcP1X5QFpeTBfzyjSkVJXntqv6HMYn0vKyz7Hu5FLmV2WzyNw4c9rLTlBVfl/lb2UTiNzJouozqBpzjqp+q04YVXae2qc0Tkx/CeU6YDXwFcJM/C4z+1xNO38HJmo77BZLgPNtixiBvumF/mnum15wzU0wm3pfn1sDn+4MfD+w3czulvRmYHFd8I5M5NZxuoykI33S3De90D/NfdMLrrkJ2tA7zG2Eewm3DF4j6bSkO4C7gRslnQTeGbcdx3GcBqmdgZvZtpKiG2ZYi+M4jjMCTT8L5esN9zcT9E1z3/RC/zT3TS+45iZoXO+0f0rvOI7jtIs/jdBxHKeneAB3HMfpKY0E8K4+dlbSckmPSDoh6TFJH4n5iyX9XNLJ+L4o5kvSl+M4jkla05LuOZJ+J+lA3F4h6VDU9YCkeTF/ftyejOVjLeldKGmfpCckPS5pQw9s/LHoE8cl7ZV0SdfsXPaguVHtKml7rH9S0vaG9d4X/eKYpB9LWpiU7Yx6JyTdlOQ3Fk9ympOyT0gySUvidvM2NrNZfQFzgCeBlcA84Ciwarb7HVLbUmBNTL+K8HjcVcC9wI6YvwO4J6Y3Az8j/C5qPXCoJd0fB75P+HEVwA+BrTG9G/hgTH8I2B3TW4EHWtL7HeADMT0PWNhlGwNXAU8Br0jse3vX7AxcC6wBjid5I9kVWAz8Mb4viulFDerdCMyN6XsSvatirJgPrIgxZE7T8SSnOeYvBw4CfwKWtGXjJpxsA3Aw2d4J7GzCwaeh9aeE55tPAEtj3lLCD5AgPDp3W1J/ql6DGpcBDwPXAweis5xPDoIpe0cH2xDTc2M9Naz3shgMVcjvso2vAp6JB9zcaOebumhnwqMs0oA4kl2BbcD9Sf5F9WZbb6Hs3cCemL4oTgxs3EY8yWkG9gFvAU5xIYA3buMmllAGB8OA0zGvU8SvvauBQ8DlZvZsLBo8rAu6MZYvAp8E/hO3Xw381cxezGia0hvLn4/1m2QF8Bzw7bjs8w1JC+iwjc3sDPB54GngWYLdxum2nQeMatfW7Z3wfsIMFjqsV9IW4IyZHS0UNa7ZL2ICkl4JPAh81Mz+lpZZOGV24l5LSTcD58xsxKeft8pcwlfQr5nZauCfhK/2U3TJxgBx3XgL4eRzJeEPSza1KmoadM2uVUjaBbwI7GlbSxWSLgU+BXymbS3QTAA/Q1gvGjDUY2ebQtLLCcF7j5k9FLP/LGlpLF8KnIv5bY/l7cAtkk4R/gHpeuBLwEJJg1/Vppqm9Mbyy4C/NKgXwmzjtJkditv7CAG9qzaG8HiIp8zsOTN7AXiIYPsu23nAqHZt3d6SbgduBm6LJx0qdLWt9w2EE/vReBwuA34r6YoKbbOmuYkAfhi4Ol7Bn0e4yLO/gX5rkSTgm8DjZvaFpGg/MLhSvJ2wNj7If2+82rweeD75ujrrmNlOM1tmZmMEO/7SzG4DHgFuLdE7GMetsX6jMzIzOws8I+mamHUDcIKO2jjyNLBe0qXRRwaaO2vnhFHtehDYKGlR/OaxMeY1gqRNhCXBW8zsX0nRfmBrvMNnBeHR1b+h5XhiZo+a2WvNbCweh6cJN0KcpQ0bz+bif7Jov5lwh8eTwK4m+hxS1zsIXzGPAb+Pr82E9cuHgZPALwhPW4RwwfCrcRyPAuta1H4dF+5CWUlw7kngR8D8mH9J3J6M5Stb0vpW4Ei0808IV+I7bWPgLuAJwl8Ffo9wN0Sn7AzsJazRv0AIJHdMx66EtefJ+Hpfw3onCevDg+Nvd1J/V9Q7AbwryW8snuQ0F8pPceEiZuM29p/SO47j9BS/iOk4jtNTPIA7juP0FA/gjuM4PcUDuOM4Tk/xAO44jtNTPIA7juP0FA/gjuM4PeW/3MrWlJwcEnwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "from PIL import Image\n",
    "\n",
    "COLOR_MAP = {\n",
    "    1: [255, 0, 0], # Red\n",
    "    -1: [0, 255, 0], # Green\n",
    "    0: [0, 0, 255]} # Blue\n",
    "\n",
    "\n",
    "def convert_rgb(f_img, color_map=COLOR_MAP, border_color=(0, 0, 0), border_row_num=1, add_border=False):\n",
    "    rows = f_img.shape[0]\n",
    "    if add_border:\n",
    "        rows += 1\n",
    "    cols = f_img.shape[1]\n",
    "    c_img = np.ndarray(shape=(rows, cols, 3), dtype=np.uint8)\n",
    "    for x in range(rows):\n",
    "        if add_border and x == rows - 1:\n",
    "            for y in range(cols):\n",
    "                c_img[x, y, 0] = border_color[0]\n",
    "                c_img[x, y, 1] = border_color[1]\n",
    "                c_img[x, y, 2] = border_color[2]\n",
    "        else:\n",
    "            for y in range(cols):\n",
    "                rgb = COLOR_MAP.get(f_img[x, y])\n",
    "                c_img[x, y, 0] = rgb[0]\n",
    "                c_img[x, y, 1] = rgb[1]\n",
    "                c_img[x, y, 2] = rgb[2]\n",
    "    return c_img\n",
    "\n",
    "\n",
    "\n",
    "#plt.imshow(color_img)\n",
    "#print(np.histogram(adv_sign[0, ].flatten(), bins=4))\n",
    "#plt.hist(adv_sign[0, ].flatten(), bins='auto')\n",
    "\n",
    "def save_color_img_to_png(color_img, file_name):\n",
    "    img = Image.fromarray(color_img, 'RGB')\n",
    "    img.save(file_name)\n",
    "\n",
    "def show_png(file_name):\n",
    "    plt_img = mpimg.imread(output_file_name)\n",
    "    plt.imshow(plt_img)\n",
    "\n",
    "output_file_name = 'adv_signed_gradient.png'\n",
    "color_img = convert_rgb(adv_sign[0,])\n",
    "save_color_img_to_png(color_img, output_file_name)\n",
    "show_png(output_file_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ricky's Comment\n",
    "\n",
    "- To visualize the sign gradient, we plot the graph in RGB. Red is 1, green is -1, blue is 0.\n",
    "- There is no change in the last packet i.e. the 16th packet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00392156862745098"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# floating point precision in one byte\n",
    "1/255.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i: 0, eps: 0.003922, y: 1.000000\n",
      "i: 1, eps: 0.007843, y: 1.000000\n",
      "i: 2, eps: 0.011765, y: 1.000000\n",
      "i: 3, eps: 0.015686, y: 1.000000\n",
      "i: 4, eps: 0.019608, y: 0.999998\n",
      "i: 5, eps: 0.023529, y: 0.999832\n",
      "i: 6, eps: 0.027451, y: 0.988524\n",
      "i: 7, eps: 0.031373, y: 0.560655\n",
      "i: 8, eps: 0.035294, y: 0.022499\n",
      "i: 9, eps: 0.039216, y: 0.000566\n",
      "i: 10, eps: 0.043137, y: 0.000014\n",
      "Adversarial Noise:  tf.Tensor(\n",
      "[[[-0.04313726 -0.04313726 -0.04313726 ... -0.04313726 -0.04313726\n",
      "   -0.04313726]\n",
      "  [-0.04313726 -0.04313726 -0.04313726 ...  0.04313726  0.04313726\n",
      "    0.04313726]\n",
      "  [-0.04313726 -0.04313726 -0.04313726 ... -0.04313726 -0.04313726\n",
      "   -0.04313726]\n",
      "  ...\n",
      "  [-0.04313726 -0.04313726 -0.04313726 ... -0.04313726  0.04313726\n",
      "    0.04313726]\n",
      "  [-0.04313726 -0.04313726 -0.04313726 ... -0.04313726  0.04313726\n",
      "   -0.04313726]\n",
      "  [ 0.          0.          0.         ...  0.          0.\n",
      "    0.        ]]], shape=(1, 16, 1500), dtype=float32)\n",
      "Adversarial Signed Gradient:  tf.Tensor(\n",
      "[[[-1 -1 -1 ... -1 -1 -1]\n",
      "  [-1 -1 -1 ...  1  1  1]\n",
      "  [-1 -1 -1 ... -1 -1 -1]\n",
      "  ...\n",
      "  [-1 -1 -1 ... -1  1  1]\n",
      "  [-1 -1 -1 ... -1  1 -1]\n",
      "  [ 0  0  0 ...  0  0  0]]], shape=(1, 16, 1500), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "# tf version\n",
    "def get_adversarial_noise(x, y_target, model, mu = 1e-4, debug=False):\n",
    "    x = tf.cast(x, dtype=tf.float32)\n",
    "    y_target = tf.cast(y_target, dtype=tf.float32)\n",
    "    epsilons = np.arange(1.0, 100.0) * 1/255.0\n",
    "    adv_sign_grad = adversarial_sign_gradient(x, y_target)\n",
    "    \n",
    "    for i, eps in enumerate(epsilons.tolist()):\n",
    "        adv_noise = eps * tf.cast(adv_sign_grad, dtype=tf.float32)\n",
    "        x_adv = x - adv_noise\n",
    "        #print('x_adv: ', x_adv)\n",
    "        y_pre = tf.cast(model.predict(x_adv.numpy()), dtype=tf.float32)\n",
    "        if debug:\n",
    "            print('i: %d, eps: %f, y: %f' % (i, eps, y_pre.numpy()))\n",
    "            \n",
    "        mu = tf.ones(shape=y_pre.shape, dtype=y_pre.dtype) * mu\n",
    "        if not tf.math.reduce_any(tf.math.less(mu, tf.math.abs(y_pre - y_target))):\n",
    "            return adv_noise, i + 1, adv_sign_grad\n",
    "\n",
    "# the target is 0!\n",
    "adv_noise, adv_units, adv_sign_grad = get_adversarial_noise(x_in, np.array([[0.]]), model, debug=True)\n",
    "\n",
    "print('Adversarial Noise: ', adv_noise)\n",
    "print('Adversarial Signed Gradient: ', adv_sign_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ricky's Comment\n",
    "\n",
    "- We need to multiple `11` (i = 10) on discrete floating point precision `0.003922` (1/255.0) to make the prediction closing to 0 (`0.00014` < `1e-4`)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modify Original Data Packet\n",
    "\n",
    "- Apply adversarial signed gradient on the original packet data from rage [0, 255]\n",
    "- Set `adv_noise` with floor = 0 and ceiling = 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original x\n",
      "shape:  (1, 16, 1500)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[69.  0.  1. ...  0.  0.  0.]\n",
      "  [69.  0.  5. ...  0.  0.  0.]\n",
      "  [69.  0.  2. ...  0.  0.  0.]\n",
      "  ...\n",
      "  [69.  0.  5. ...  0.  0.  0.]\n",
      "  [69.  0.  5. ...  0.  0.  0.]\n",
      "  [69.  0.  5. ...  0.  0.  0.]]]\n",
      "\n",
      "\n",
      "Original y_true\n",
      "shape:  (1,)\n",
      "dtype:  float32\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [1.]\n"
     ]
    }
   ],
   "source": [
    "original_packet_generator = PacketDataGenerator(v2ray_packet_file_list, shuffle=False, rescale=None)\n",
    "x_org, y_org = original_packet_generator.__getitem__(0)\n",
    "\n",
    "x_org_in = x_org[0:1, ]\n",
    "y_org_true = y_org[0:1, ]\n",
    "print('Original x')\n",
    "print_detail(x_org_in)\n",
    "\n",
    "print('\\n\\nOriginal y_true')\n",
    "print_detail(y_org_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Masqueraded x: \n",
      "shape:  (1, 16, 1500)\n",
      "dtype:  float64\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[ 81.  12.  13. ...  12.  12.  12.]\n",
      "  [ 81.  12.  17. ... -12. -12. -12.]\n",
      "  [ 81.  12.  14. ...  12.  12.  12.]\n",
      "  ...\n",
      "  [ 81.  12.  17. ...  12. -12. -12.]\n",
      "  [ 81.  12.  17. ...  12. -12.  12.]\n",
      "  [ 69.   0.   5. ...   0.   0.   0.]]]\n"
     ]
    }
   ],
   "source": [
    "x_org_in_masq = x_org_in - adv_sign_grad.numpy() * 12\n",
    "print('Masqueraded x: ')\n",
    "print_detail(x_org_in_masq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-normalized prediction:  [[3.5589628e-07]]\n",
      "Original prediction:  [[1.]]\n"
     ]
    }
   ],
   "source": [
    "print('Non-normalized prediction: ', model.predict(x_org_in_masq * RESCALE_FACTOR))\n",
    "print('Original prediction: ', model.predict(x_org_in * RESCALE_FACTOR))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalized x:\n",
      "shape:  (1, 16, 1500)\n",
      "dtype:  float64\n",
      "type:  <class 'numpy.ndarray'>\n",
      "content:  [[[81. 12. 13. ... 12. 12. 12.]\n",
      "  [81. 12. 17. ...  0.  0.  0.]\n",
      "  [81. 12. 14. ... 12. 12. 12.]\n",
      "  ...\n",
      "  [81. 12. 17. ... 12.  0.  0.]\n",
      "  [81. 12. 17. ... 12.  0. 12.]\n",
      "  [69.  0.  5. ...  0.  0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "# min(max(x, 0), 255)\n",
    "np_floor = np.zeros(shape=x_org_in_masq.shape, dtype=x_org_in_masq.dtype)\n",
    "np_cap = np.ones(shape=x_org_in_masq.shape, dtype=x_org_in_masq.dtype) * 255\n",
    "\n",
    "x_org_noramlized = np.minimum(np.maximum(x_org_in_masq, np_floor), np_cap)\n",
    "print('Normalized x:')\n",
    "print_detail(x_org_noramlized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated prediction:  [[0.00043797]]\n",
      "Original prediction:  [[1.]]\n"
     ]
    }
   ],
   "source": [
    "print('Updated prediction: ', model.predict(x_org_noramlized * RESCALE_FACTOR))\n",
    "print('Original prediction: ', model.predict(x_org_in * RESCALE_FACTOR))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How about the whole batch?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch version\n",
    "x_tensor = tf.convert_to_tensor(x, dtype=tf.float32)\n",
    "y_target_tensor = tf.zeros((32, 1), dtype=tf.float32)\n",
    "adv_sign_grad_tensor= adversarial_sign_gradient(x_tensor, y_target_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_tensor.shape:  (32, 16, 1500)\n",
      "y_target_tensor.shape:  (32, 1)\n",
      "adv_sign_grad_tensor.shape:  (32, 16, 1500)\n"
     ]
    }
   ],
   "source": [
    "print('x_tensor.shape: ', x_tensor.shape)\n",
    "print('y_target_tensor.shape: ', y_target_tensor.shape)\n",
    "print('adv_sign_grad_tensor.shape: ', adv_sign_grad_tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adv_sign_grad_ndarray.shape:  (32, 16, 1500)\n"
     ]
    }
   ],
   "source": [
    "adv_sign_grad_ndarray = adv_sign_grad_tensor.numpy()\n",
    "print('adv_sign_grad_ndarray.shape: ', adv_sign_grad_ndarray.shape)\n",
    "batch = adv_sign_grad_ndarray.shape[0]\n",
    "\n",
    "c_images = list()\n",
    "for b in range(batch):\n",
    "    f_img = adv_sign_grad_ndarray[b, ]\n",
    "    c_img = convert_rgb(f_img, add_border=True)\n",
    "    c_images.append(c_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_image = np.concatenate(c_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(544, 1500, 3)\n"
     ]
    }
   ],
   "source": [
    "print(overall_image.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAACcCAYAAACENOsLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO19bcxuaVXedZfhw68yM6CUMmMHWqIhTSvMRIZoGiMVR2ocmhCLtTpSzCRaG60mOmhS0/5S26iYNuJEbAeDCEUpE0JLKdC0f6Sc4wegODIqdGYCDKigLUmVevfHs/eZdda5rmut/Z6P9znHvcjD3vv+WOta6173Wmvf55n3GXNO7LTTTjvtdG3RXzptADvttNNOO1162oP7TjvttNM1SHtw32mnnXa6BmkP7jvttNNO1yDtwX2nnXba6RqkPbjvtNNOO12DdFmC+xjjjjHGA2OMB8cY91wOGTvttNNOO2kal/p77mOMxwH4HQBfA+BhAO8F8E1zzt+6pIJ22mmnnXaSdDkq9y8H8OCc8/fmnH8K4BcB3HkZ5Oy000477STousvA8xkAHgrPDwN4fh40xrgbwN0AgM/DrfjSQ/utZ4Gzt3LGa9+tZz2AOL8ay+ZmDJlHxsfGV2Myqf6sc0dOHBfHVDijnluxRrnsXhEbs5VHB/O1TpV/beEDXByvS4XlUpHbj5095Ph14tUWfN37c/jO4pNzzi9kfC/HscxLAdwx5/z25flbADx/zvldcs5tY+LM4X4OYBBIsX2OMFeMlbLmY7zcOEeRR4e2jGXzMt5sh2oMu2cyon4VrixXYWdzKixOjtKT8WA+k/uy3h2bMx6da1fPjKXCX/ljJd/ZQ42PmOJY1u/8ydlonXtS365kZvlRXtar6/PVGIeV+WJ+vkDfgbNzztuYnMtRuT8C4ObwfNPS1iLlhB0HifdqsXKb21BOTseRmHN3HTLLUht0jn6AdbJUP8Ob5XY2VrRzFVyYjXNbXleXqJ0OeRzDHftYgRGfWbsqKFgQ6eBSvuECIrOv01fJZqT8MvapwNVJhlnXuIbZHhVelkAiuTaXhBg+1h/nqUSi+jOWii5H5X4dDv+g+kIcgvp7AfzDOedv6jm3Tayl+0477bTTTk0aV65yn3N+dozxXQDeDuBxAH7OBXYAh0OkM4eU1H2VW0lVVmr8Ooa1ZT6u6uq0M1ydV0ynT2de9eruqhulT9Xn7HKSNcq8nK6dNavWi90rfK4a3WpvhqMaV+Fx1Wllr3WOe7PKc9k4hTGT83vlRwwPk9HZU27vKllKrnrzqHRSsYhhzORc5nIcy2DO+TYAb7tk/IjBM3UCQHxt6shwGzS/GjMsrs/Nyc/OufOzc/DOq5zTU+HboqNy1Cp5rrq5MW7tlVy1yarA5JIsu6rg6RJN1iHPVTpUSVStQ9bf2STiYbpmvV1R4LAoezCbOl0YD1V8uISX7cv8zu0XFYs6NtlKR/1fqDLDqTFqoboVVA4irD/eRznO6TNONiePzzpF+avsKnAqe2RdMm/VVlUqTJdOUGXPSjeVgE9KbMNF2yo7Z3KbktnABaTsVzkIKT9WFaPzl4xZ+UFVHKjkowK04s18XBUsLBEpfdQ8tYcZT+fnVYypigOVECuqxlzyM/eT0BgXs0V32mmnnf7CkjxzP47K/VYA8/BZLo/dTP6cx7F5bPw6hrVlPvma8TgeCleeW+l4kjZlJ4Z7y7PC7T4Oq7LjFtnVmlXr6eQ5nJV9q2vXnm6MWm/W5mzKbFIurBjnbNvhweZsWZMO7622rnCqMWyck6l0sfMNXZYz90tN6jU2j3GvapGq80Q2No6pzs8Y9kq+OwpSr4wOvzsDrF6jKx0Y787xmeKr7KvOPOPY6vVcHREwvJ2z4nUce/VnxxcVn0xMNjuWql7l3Vkum5vnVb6gjlw6R3ZZfmzPmJidIw/my+x4tTr2qI7b2HOc2xmbZaujNydrC10VwX2lzlllHBsXuTrfVMG1G4ArXszh3PmaC84MX3WwxRyfOVAnQMY+Np/p4QIeC7IdnXNbFVjYuIxT4VcBJeNWWJ1/ZVydAOSCdLa58zO1D5wfKJ5OFyVT2aTSu9OvijGmQ8bMcDEdWIBWspR+KrAzfBfwNEngqIM7U4wFUzaWjVnbVHWisuXWLLqFjwoUW6nDw1VlrHrL/dGZgTqYdvhVeCpc1diqrQocud1t0k5QUP3ZliqAqQDu7jvPLHjn8Z0xaizDHuWqpKJwsL6OPsrObu1colMYKl9yCU5RVWhmOo4z90BdB40LvH5OUlmqdiabbXDXn/HlMSwZbeGjPhkPs5XSXWFw1U3s6+rX0S3LqajDh/GNCSvr1MFQJVbnc1m2CxZVv/J/l9QrXIxvh3e2H0vmshpN+qikvNXuWS7rY37hEq1aOxarWJJgV5doMjZHRxfcK3KVnDNcRS7grc/OObJMF0RdRcFwZUdmTqH0zMHZyci8VMJUvNRYNi5jd8GkG+gVL6e72uhxnvOFiEsFkyi/CuRVkM64FL+st/OVjh9lO7G9xngyLBWuTlJVyYLhY/izzMqmyq9YX7ZBTG7O9lkHtu5KP0ZHF9xV8HaB0WV0VSXG+azaddWlkhnHVBuo2nSqosl4lV1YcNuijyKXQFVVHO8V3m4Ad+NYQlW6V3OzH2R7Oj9gQTjjjnZgtoi2VT7J9GSJUdmMYYh+Ua1Jns/Wl+nexRfHMJmxP69xFUwzRoUz65ht43BWGOMc5VMZJ7tXdHRn7lXAchUDm+cWN28glVjYc57fnaPm50VTzlnxuhTjOroCF+JkiYTpkpOhqsKYLzj7dHF3k2gkFnBzG9NXYXQBk9lFbXyFUyVhVQQ4GUrHSl4ey3AyngyLS3oqaFb7Uclm/uv4Ot9V/p1lnUSHOWD//MBRV+4rsYzKMp6rbDqkqih2H9tUVt/SX+naHcv6WPWn9GHkAoqzt+KvKmmnY4Wnow9bS5c8YvCpAq1ry8GQBYMq6GeszFaKmD0Yr87e6ySlrt6ZXzdBRD6qcFBzM3+ll0tIjJfSP893sliSZIG/G+OOrnIHuPJsQzDqVrBuXifIKYeqNidbvHjfyfZZJsO1xU6swmQYFC4VsDq2YLKZfKVn7lOJjgWCihfDqvqj7Gj/jElVoiqAVRiUjzBdu7IVn3hVyZDZJY5j9lB81ZhsI2YTVxGrAO2Sdx6TceXxav84n2c6OPxjAq50P7rKXW1m1u/G5fGsamP3ebHygmXnyo6pggwLKMxRWbbPH6bfes/kR0dhFUvWQQUjxtfhVZuD2VnpEZ+ZvIyp2rhqEzMbK13UWkT5jBfDqvwxj3XYO/7P+GXMbK7TkQXQjk9nGTk45uCn1lftz06hlgN4J+4wn2X7xiWiKMP5BwvybJyjo6zc1UausrCroFh2rSoN1acwd8a6jbT2V44W25XcLgZVia2YVFUU711SyzxZQGCBPN9nXp2KUfkG26BMJ0Wu32HOmCrenWqS9bkgXiXRPF7xr/wy3rP1YJhYsI7Vqwp0ymYdvAwH00f5jxrL/EiNVzzZ/roAj9GprNzHGD83xnh0jPGB0HbjGOMdY4wPLdcblvYxxvipMcaDY4z3jTGeV/HPxIysspWqtFw1c5LqxcnOONgn81FjmDOoiqiqkKoqkMl2gUNVcUxe5lNVYUxnNidvHoWR2T3zz7jYM8OU8TBeLkgw6mLIVWK1xrFdBRnnA1lPJouttdobSn6WUSVt5R/s3uFguFhiUPPUnsiBne2hyk+ZbzIejjrHMv8ewB2p7R4A75xzPhvAO5dnAPg6AM9ePncD+OkG/5K2Vq9OaRfEtxg+y3Kbt5tQlJyuLFcFqc2jAmwnQVZVmtpILFh0kmsOpK4yVBVp3DQqeaz3zgaqwsr9VbHCZCpy4+NVfdw4pyNLOMq+TF62c7SX86GqoGD6ZGyduVVbbnf2XSn7GfMXN1/58hY9W3/yd4xxC4C3zjn/5vL8AICvmnN+dIzxdAD/bc75JWOMn1nuX5/Hef77z+zttNNOO22nS/8ze08LAftjAJ623D8DwENh3MNL2wXBfYxxNw7VPfDFAD5ySENV5b2OcdR93XZzXVXpXvUVMZ7sFdDNczwy1i5vRUpm5pkrsGw/V9GwZ9XWoWoNdqqp48vHQl2s0UezX3f2kJPbOSmo+KgxSs55+Mw+ueh/UJ1zzpP82Mac814A9wLAuO38+Z0gtpKTfDFBfpWb5bijDBdQcwDMfYxHPEKJPNTrLMOU8bH51Wt3bqvWgummxsb7fJShgr6zA9Of2fAkpDBl2Yo/8w/lYw5zdx0VD+a3zJ86vpexs/kV/q2FQCcYqyJH+UunwMj+zfbMloItH90wcn47h43tJ/4q5MeX4xgs10eX9kcA3BzG3bS0bSYVsNg4dX7lAkHmzRaMBavcFxfK6eGCperPsnJbDpCrjtEGeWy8Mty5X2FRlXqll3tWeFiSVMmy2qRxvSq7dsbENjZOEVuPiE/xUPqvcxVW56fZJsqvFC61Znn/VTZy66/8K8tzmNiV8WMU9xaTweyjkma879jUxSFHJw3u9wO4a7m/C8BbQvu3Lt+auR3Ap6vzdgDAWRxS0Dh3OWSkwT+imX7W8Wqe6o+ymGzVn8d0eXfkOnnONh1DMcwnue/0dezXkcd4uXHMHqpP8a2ws/Hdta3uMwYlR+lZYXXjO/bfso7VvK5tO5g7V2Y79VzpuNVuSoaSdx4uQ+WxzBjj9QC+CsBTxxgPA/hhAD8C4I1jjFcA+AiAb1yGvw3AiwE8COAzAF5e8Qdw+Jm95d9TqyOPOEaNq15rqZ6N16mK2Kvm1rm5LWPp8lavxsouHbuxajFiUq/aFX5VcXaOWdg8hUnp7GQyjE6Gq6irClvJYGOYnZkenbVWvCo/q8a5/bDFDxi/2KdwbFn72NbdhxXOLfMzOV85b5zlcQw/kH3bmDG4d5yqE0SszIsMVGozdB2Z9W3Rx8nITs10ZLyq5OQ2c6XjOs8FxK49mTwX7NUmZm1VAM59FbaOTTt+4wI/C+YK35Z9sjUwOr1cEq3smfkxjC65RhndJFPt/YzXJZzK/6pixRY/R/8D2YGqqim2xbHKYfOY9VNVfplfXHAmW81b26K89d7psBUbw6n0je2ZP3PiKIPJZcTkxT6VfCLuLD8+5/YtdoyYVEKMts54thQRjDfTO/pWx7bMtxh25htbsFfFDMOW7zuFSl4LxY/5rtvPLIi64Jz1Yn7H+Ff7ie1dxmMrf0dH/ecHVmIO5jKkqr5U9lftDBe7P8mcDi+GsyNbbfZuMGYbivWzJLBSx55unRzOThBwvFQlVSUJFlBY0lI83AZXPPNcFgjceBbombx8ZbyYrE4iVdgAbUOlR4enC+LVWqsY42zMsDNb5rZqnfO6bKWjC+7MkGzTsLZMWwI5c/zYrhZaLXzuc+OcDq6iYHMZ3i1JT1VYlT2qyk4FZaYDk99pr9aBrT1rdzo5OR0/cLqs/nwxvsJsvoWf8yfmz1UwZrZ3CcX5kNKTtbnky+apPoYhJw9nYzdG2cElpQv4XijyHB1dcAe8M6pxynnUvEwuEcT7ytniYlbVlSOV0Bjv2KfatsjfMm9rVdEJtFm2SwyMN9MhBu48x7U7XrGvU2TEuSwQMR0qfTJ1E1fGXq2j21MuMKo2hividn2xTa2F87M4hvFyyYbxqGzL9M6YnT4npaM+c6+UU1WxWsx438nSbDM6vgwXk8H6Ou0RN3OY+IlY1GaOH+bQDBNLOFUiY7yzbWN7VQFlynhcxZWvnWpPyWUY4yfzUGuY+9l8hp2Rq1Sj3Mxf+YaT5fw7ylR7S+FnQVr5WF7LblJgGJTfMTsy/1XFCksUysbKRh3/y3R0lbtz5NznMqLb8N0sHOdnjAx3dc/6GJ5ONVnJUTKYHVx14io2V/Gx9WFjKuzsucM7tncrNJfgYpsrKrJ8layUjMwzy2UJscOXJa4oL1+ZbMVbyWWJsxpX2ZTNY/ycHaMcF/RzHwvOkZw/KRxMntKHzjcF8NEF9y3Ow9pY5o/UeRvoBKssXy2koiqQ5HFOJ4Ut9+VNvAUrk6PanYN3bOiqK5WomW4uaFUJtLJzlZjy5u9ic8GX4VR2yLyiDFVdVomc8WDtLlg7nZhuWSenf8ZW+b2yO/O/rv0d7mofKKwKO2Bj+/Edy3TJBdOtVU3my8blReo4MHvuVsuV41d6ZIzOeatqkfXFT25n/LdU10o3Zzu2UdkmYVhYgFXrwkjZqtK72uwqMCj/VhWn4slkMP+qio94VUEpVrud9d1S5LHkUlW/Wabjv7Vf8VYFReTrEs7WQvIogzvb8LkvPuf++FGbIDtvFTRVMHOblFWZ3WqKyYif2MY2TscGEYPaWFGHjFdVFipRMB2d7DiH6cvs5YJXHK82f+ULcS7r71SIOQkymzPdnJ8pG+d1yrzyPJVEVHBxtsgy1+e8L7I8ZRcn3+07pp9au8i/SqgKW+bhkrEK5CwhuljC6OiOZSKxBVSVwfqcx3c2e64w2DiVbTtjq+SUMbNNXFUdDt8WTNEWLOAxPiqYKR55HMD785pE+W7NtthCVXQqeFTr0En4WUbmlXk4fhljZXdV1DidlB4MZ6bO/lTzna1cglf8FcasDxubA2uXlwvE1f5V63PeHuAQABxh5e42ZrWYMcMpg3YqjM489ZyzdW5398wR1RwlW1FnTJSr5jE87FltKlV9qcDucKkgx/AqzCowx+qJVXKMso4Ke5btMMT5VbXmfCdjUVgzFuXXWUbmlcfkeU63KqjlfZ5xV/gzdsajigdZR6ZnXmMXF5xvVjZWdJSV+9YA1Gnv9scxKuu6Rcgbs7sZmDy30VngdMEltquqwlXWqgpRidAFxWp9lWM7bBljVXl2npl9WELIc51NHa+sM1svxquDPbe5xJjHMp3zGlfBWSVmNjde2TxlC+YHmVcV4J2t3L5yuJQdlNzcrvy6qtyP4w+H7T+zt9NOO+10ArqIn9kbY9wM4LU4/JTeBHDvnPNVY4wbAbwBwC0APgzgG+ecfzTGGABehcOf/v0MgG+bc/6qFXLrWeDMIQdVr57rGIvZVDQVqYyv+K/96pXYtWWMKsuryk0Rq0gcj26F4vC6txdHqorcum4Zz04no64PXCuk9ipw8Tbo7ln2dljdn8Nn9knnzP2zAL5vzvkcALcD+CdjjOcAuAfAO+eczwbwzuUZAL4OwLOXz90Afroho0UsCLDP+vqizuHi1clgRyuxb8urmAqqkX/1is5ey/KRzBadmbwuMUdjr9bxU41TWBUPNo8Rs1PuU3bcavNOP8Pn+jPO7vhqDOO/lS/bZx3b5fluXbbqXlE+6mHYGH+3xxxvNyZjyP3Mn1TRmakM7nPOj66V95zzTwB8EIcfvb4TwH3LsPsAvGS5vxPAa+eBfgXA9etP8l0snfT8io2pxrKAzgKO4t/BUSUFdo6Y57Pz0Iqywyi7qkCX7aBkxGvU52Iq+5y8MzYVWLKu3c3MzpfZOrDAk9cv42cyFbZ1LsPD5rBiwQVQliSVbRmurAvzV4ZF7SOX3KvgqhJClZjUWzuzkUo2WWdnO7avWKUebcD2oKJN/6A6xrgFwHMBvAfA08JP6H0Mh2Mb4BD4HwrTHl7azvu5vTHG3ThU9gC+GBgfObSf+z+DQzysc0doaCf3PFY5spCpno04O+48Xcz1PDbj/LlKzgVimZ2UzAK3wn/BVNWn1pPxMMKlD2SbNeZn3BfcKj+rZDs5TJaQbfEZv87tajzzDYmB2d7tw46fNv2HPJ7XIbEIH1d6XyBH6GDtZvZVyz/O9WlPbgf3McbnA/glAN8z5/zjw9H6geacc4xtJ1RzznsB3Atg+SWmA7+cOencaECSzasKn5GqENmV4VHjVMVd6dnVkVUa+Z5dt9qh05fxuNfIao2YvZWuWabT1emv1kVV0a5S3mp3RR1M7u3N6a982lWGzrdjG5Nb8Yt4md3cPmIylP3VG4DyObef2NxKp4y3spXdr7qr9z33McbjcQjsr5tz/vLS/PH1uGW5Prq0PwLg5jD9pqXtxMReq4DzjaBe7+K4bsJYn/NrddcpMi7Gg+HNC89e0/LrWZRRbdRu+s2vn+61Wb0usnmZr+OXdYhjoqxqY1Q+weRHGR0e0U557ZVubGzuy/pVySGvhQp8cVz2aXZleyPan/mi0yPyZ/ZmONS8Lb7NfDVjVPs188k8u1jyvmY8V9nM9luoDO7Lt19eA+CDc84fD133A7hrub8LwFtC+7eOA90O4NPh+OZE5IICUDuGSw7sE2UyWexaObDapBVGxi9iX9tdUM/8O86oKhOnTw5yruLNCcoFkywjt0W+LPAoW+X22O+KhYhT6ZkxVoE2Y2P6uj43JwesjGcdp3TPa8/aMjZFbr8of2Q2qPhnndh+jc9xXPYxhtn5MOOb56q9wRJ8NyFcyK/4nvsY4ysB/A8A7wfw50vzD+Jw7v5GAF8M4CM4fBXyD5dk8G8A3IHDVyFfPue0X2LfeqSz00477bQTAPMD2cfxHzHdNub63zCpCo1VbCwluLGuqmCVquLHKgJVfXUwVnKUrh1SFaOr3PIYpy/DX9lA2SX3qzcHRVvXjOmm/K5bLVe6sbkVfoalU7UrfSue7o2EjanecpWNHY8te3xrXyfGdHhFHmrvrOT0czGF+eB5fE1wvyr+/ABTTjlF5bwqQLB+dc8w5rHsdds5rHIStbDVa7ALYm6sCqjsNV05vsLvHFmReiVXTs9sXenM+LH1i7IULhUg2PqpYFAlipOuS8bDcMX5qnhhmJUcNa5KwJ19rAJhZQPG2yXBqAOTycarRJb5533H7FbtEUVHE9xd5mQBx23oddzaF+fF+06b4hGpE+wUD+eETk4nuDKdWOBSDprnMF4MW5xTYVIBVAXejCfyZrhUkHc8VHLoBM5ILGgwWbEttrsCopJfBek4hgUWh9XxcXsr81XyMtYte1fpxnBUyTW2x7jjEr1qZ36jYp6LYxHLmID7uszRBPdOpnVKs4ChApjLiFv6TpJhHd8KA8vyXRmunyVM1u8SqAusijr2U4mEjXGkeLuNmgObKj4UrpO0qwRSBZTOs1q7qliI450NWAJ3MlxQVvLjOIVLJbP4rOa7GNP1h6oAcEValbQij4qOJriv5DZLR0lVgcU5jNhGjzzZAneNXMlTPDqVgdJTOVCnmooyVdCPfJ1DKhupJKyqrSopVzZU2FhlxuaqzbgFSxyv1sf5uFo7tZZMbp7H5LoqVmHp2F/poGzq/DtjYtg6NlG+meewoBzHMD1VknFFGrO34jUHLv577qdNW6pBFRC7crqbRAU/h89Vn2xs3Hx5I+Qqab3vVJIZi9tADIMKRJ0KUG32fGWBp5Ps1ytLvrk/464o2injjB/mD2qN3EbP8iqfiRgiD+bTzH/Z2GzLav2Uzmpe7Gd7a70qv85JgiVKx8/5RLxXxYVab6Z/Z/93/JKtE6OjCe7RibobLc6Lbbm9ChKRmEOohJE3kuOtAk0eU83tJoWInQWIeGXVI9ug8ar6Myn7sOonB0i1UTJftfHYc+ahcKhnpoMLcOu9s083IDs/cEHGjcsYWRBVSZSNUbjzWLWOUU+1X2J/vFfrmWVmfOw+ynK+x/TN+rB753NZT4WvigVHcyzTMZirMF0QctVRlKE2apaxJTC7MWyBWVWxlX9nY7vqqlONKdl5HaNOSl+1rmqtI2+HSdFJdFHVZexTbapgYfM67R2dlN0zdseX+YOzebUOVVJR+0Pth7xn3HiHyQVJtu4uPin7by1AsixpOw39eCr3laoqJ9+zyifyYnMdVbKZPNWXKwrXr3gxPmxsxpqrBJf1VXDKSTXPrzZ45pt5xjZVrbFkzUjZl+nM1lgFkPhRWHIQz+PUOqqkt14zPxesGDalrwtCFf+KlO/ldY3jXWGg5kQ9mByVhLNMpZ9LxoyyP7PxHRu6fev8l9HRVO5dcpWnMibbJG6DxrlMfuUIThbLxs5pnCxX6bLNUTk/451tkoNGp7pi9mZBpbKH0ilXxkqWqqQq+zufc1VtDjoOY75n8rf4kUvMaiwLLGy9XQBy68vmM1uwgJYx536XANUeYjZk47OPMb0ZqaSf+eY4pbDmtoqOKrirAOsW3QUkFQzyfZzj7hmGzkZmc3Nb11Hy2K7jVFWE26hOB7Vh2RiGo2pjydetacbpfEKNUaQSVSRXJDDbdWRUPphls/s4hmF3Noh9LPAzP1OJNPPN+4bpnPcU8/duEHT+2o0bcZ7bM27tVXs3KZ/rM/56VMFdZUS1AJFckK/GufmZD3PIzkZW+kSZrr1jG4bBBbOsT8ahZLrKpwqALLioja/aOtXNSls32BYeF5OsFQ5WdHRwKWxdHBej45YxHYxVO6OOfVh7TiwsuHYStJKl1jOPYXJcMdp5czi6M/eV8kbOAT0vSPxEHtViVAuYq4c4RlV5VeBi1KmycjBVSU4FxlyVVJUQc6hONR95ss86RyWPyHNrcMnVftY9trs+d1V9zAaOGE/lV269mayOTo6P8kOnEwuQ3eTO1kp98rwKl9LB8crrUK278jW3nivFfaF0rApVRkcR3G89e2Eb28DVJlY8IrGAxhZSBY54rSoDFhwryglEJZXczmS75Mb4M1l5rhrDqp8VhyMlW8mP93ktqw3F1jlS5KN0UDjZxlftVTJQxUW8qvWN4zuFTLWPVPJlulS2dW3Vfl9J+WSUWa1FVfEyjFVVzTC5wKxkxXsXvzrU+ZO/TwLw3wE8EYdjnDfNOX94jPFMAL8I4CkAzgL4ljnnn44xngjgtQBuBfAHAP7BnPPDXsZtE7B/FXinnXbaaacLaMi/Cok5p/0AGAA+f7l/PA5/x/12HP6W+8uW9lcD+I7l/jsBvHq5fxmAN5QybsVc/zdx+MT/sec8js3LY92H/W9tV3jiNfJxuPKciqfCpOzg7NS1hbNR9azkOJ22rmsXJxubeXYwujVTMtWaKfld23bWtloH5W8Kb2UXZ6vOmnfkVGtS7ZWKt1o7N1/Z1M1T667wMVuehxk4o+JqeSwzD/S/l8fHL58J4KsBvGlpvw/AS5b7O5dnLP0vHPEHVzdQdb7HjhjYPHd+u7arsz133pVfz7fv6u0AACAASURBVLpn9aq94hXHVTqrYwl3zePis3uVVWvhjn06Z7fumIKNZ0d3rG0dq+waxzFfYscJ3TNf11b5jzrSUccd6rVe+Yo6V2bHVGqOOv5g+yrfd49y4r07MmJ88l6PON2xTnXEyo7h2LGVOkZTum85hsnU/Q3Vx40xfh2H30l9B4DfBfCpOednlyEPA3jGcv8MAA8BwNL/aRyObjLPu8cYZ8YYZ/AJITcFH7X5soHYeZc7h43X2B7lqkTCFos5QXW2mSlvGIY5jnPnetkeDLfSlwUWprPaGPle6aQoB1NGLhBHeSzZRBm5f+1j47KcTpDJuNSZMsMR8eXArwoINqYqghQ/tscqu7B94+yg7Mrs64qGKDf7ck7olR8qn2fzM2+lp0vqLDkqjBX21lch55z/D8CXjTGuB/BmAF/amVfwvBfAvQAOP7O3KrQOYJti+b845oJ7wueCeRcwJSIbeCQ2dSU4GQ+FxcJX+hP5SuZmftmmRG8pQ+lP5tH1VkRkKnkXyOhiFPaUuldjDT6Gx64lwSD5NRrl3mFr1PC9LftEQtyyHxT/xIfpJ+OB8hc2loy3sjq4JejzadO3ZeacnwLwbgAvAHD9GGNNDjcBeGS5fwTAzQCw9D8Zh39Y1XQrgHn4LJfHbsgnPlbzGAvWnue4q+Ofxzne1Tglzz0rHk53Zctqbqevo5NbK7fGypas7yT6uDV2+rLrlnWo7OBwO4zKdmwdLkWbwq7WXmHrrq/i2V1bpVPX3oxHd153rS8YZ6gM7mOML1wqdowxPgfA1wD4IA5B/qXLsLsAvGW5v395xtL/rrnhh1rVOWG8qnn51cydO1av0e4IQ8llr6fslTbPUzJOcj6nzindUVLEmY8n8lymszvmyDLyq2m2d+arbOTW1h1pMbu4Z/aqrY6J2Fqr4ynmX+5V3sllpMaqIzTVxo46lX5ZDntmfpr73N5U9oz4qqNKdsTkdGdyGM9qrdgRj9pfai9voc5XIf8WDv9A+jgcksEb55z/cozxLBy+CnkjgF8D8I/mnP93+erkzwN4LoA/xOEbNb/nZNw2xjy7IQgzY+ezqioRMDnuTJmd4XWwsr6MswowWUenn9ODyc5zGe58dTwZOZ3iPGVnJp/pUdmp6yuVzlvWQPU5PRQv5SfOn2I/4xHbOn6qxlS6dvaHs7XDlHnE/qx3Z91Vf+QTebG5XfyKv9sL5403P5BdBvcrQeO2MdevuXcCn1LWLXgVGKuEcLnoSsi9VDKqDbOV10pb519Km7FNf1JyAexS8bgYnCcNsGr+SYqvTvC+FGvQsePWIu1Szq3sDPiYda7PBPej+C9UFbmNF9vWj9vwa1/XefKYVUbsi2NYP2vP9xFP1KXDp+KtEqDTX/FlWLOtHB4nO+tTYVRvHI6Ps617Y3Pk3nzU3Ng+x/kfx6PzdsR4d7FEu2RcbD4LyuuH2ToGKbcXo88q/E6vLFPNy/2solZ2U0VNtXaqind2VjGnU9wcRXCPf36gu7FVNsvjlGOyNpc8nKxs8MppVWBklOUr56scUQX6KEP1x2f39uOc041lwSQHiewXDLMLSp0glfkwnKo/j1OJoioIVKCPfZ0kkrEw7DmQZ+xZDvNhFdCZvZmN2HqpAkzt/a4vsn6HMbcrX2D9GS/DreTEPa/kuhi50lEE97O3esPljL/2MQOoCrNbUbrKLvLPpAJ6XnwWnJmDs42VbcCCft5ccW50GlZ1qWDCdMy2z0HLbZI8X1XalVPHNlYUZPvkdpYwFJ7Ml81h/SwgMf9WOma5LKiydc/jHVX+r/aCkpt9LfNRfuj2LMMc71mwzu1Kp6wb80G2Rowf0zfyZ3uDJcbcx56rtT2aP/m7ZRNVWZc5I5uTF94lmM79ScYq5+nq6jBkctWO2wiKv8Pn1is7pwqAbE3iHJbwKn2dDViwZH0MezUnBzuGOetTBQ+FObe5gJP5VDIizow9znVrqngznKygcGNin9v7jHLBUfm848GuCoOKW0zWBXY3+hxNcI/kKhfW1t2M1eKycVUV1qmMtpCrGhQOtdncxnd8MgZlQ+W0MZhl/Cx4KWx5XV0gd36gxrBAXQUCVcUyv6kqTxb0lB9nvN3kp7BnHG7PdX3dJZg4JvJSid4FNWcPF0Q7OFi/SrbK9i4ZOb26SaUby47iWCaS28Brf36uAjCb5/ivThKDm6tUquzK+ionZs6qApXafC7oq3ZGueqMuOIYRqySi9esbxXAs3wnl9mQyWAJh/GOvsASA7OLW+dOYaASEePB7MPmREwdf1QJjY2LdleFh+KrAiTTifkTSwD5PrepGMHmV8FV2c75tfMBtdbdWHZ0wZ1lX2d0VV26gMf4sIVTctc+FqhU9lX82dwoI8t3tmD8mE2UPplYEKg2RR6T9VVj4zNbI2VbFSSUbswGLmhkiok/6uj0yzLVJs2Jg/lJJmaDzFsFboYpB1O1vzpBie0DlTCjLLYezC5MX2ezbN/cHvGzdXRJIs5zQbzyq4wrr4OSzegoj2VcFeQ2oqpuMo/cnqsON4/xqGRUmysHCFWNKDwdGWpelFkFQbcJutVElqnkM565GnSbSfFgGFxAV4mDrZHbnAwPC6hZPyWLYWS2UcVN5ucKi6wP63e+oAqurs8ouSrBZjwVMX9yRR6LHS7pObtl+zNZSpc5cHWduVcOmSkaQDmRGs8WdMWg5jMHrTKoIlYNVZuzmq+c21UMKmixsapNOXwew/RSumZd1GZ1Cb/Tnue7ClLJVXpVeFmfsgWzp+LLxse+td0ljq3BM2N3/qcSkQtkTEfGW8msdGDYVdDN45VejLcil0TUHnfR/eiCu8t8arFYu1tAdo33ahFUYFeB4CRBn/F0md/xcbizvPXeja2SXZzP9KhsxNpcBaYqqwq3qp7UurINzqo7F3SrAGmrs4YfqcRSJUSno5rv9O4GpUpfZy+VVLM/dAojJb8qrpydquKss4erpHOu3fA4jj8/sP/M3k477bTTCUj/zN5xVO63ngXOHHLQ1tcXR64i6c5n8lxlzZ4rcpUGfRVr8HOvjkx2pkofVfG5V3MlR9l3K53kTWmnv5i0dY9ezjmuwldv7+fGX03HMozUa44KOOp5nVO9amaesd8d88TjBxcQmezcXx0rKdz5lVclIud07owxPqv7yN+91qrjGpWQq1d+dyxQUUfnKmlVr+CVTgpDxWerTyu+jpwdWfGxtajq6OR8OcpeqeMPXV/Luql25wPdIosdMeVx53hqlv2vQi4/tfdrY4y3Ls/PHGO8Z4zx4BjjDWOMJyztT1yeH1z6b+nKyAqck02CwDquUj6PqTIpk6WClgtgqs05uwteiufWhOQChsPQcX7mkMz+mW9OkAwvS0gssaoxWc/83A2eMYkwHsxWTJ7CwfqUnZ3vMf9kSTTPyWNZkmNy87rlOUyvrIPjHflFyjo6f+vwW/ud33T2YcTB/CN+ot7ZFl35jLZ8z/27cfiRjpV+FMBPzDn/BoA/AvCKpf0VAP5oaf+JZZynszikoHHu8tgN+eQmNtdMlzzyNd+7vgpzd5zC0tErj1E6dfSvbMX0V/KrOcruW2zQXf+ufpXMSp7S0a2ta1O6dOR09emMZTLVXGVPxS9/3Jo5+RUG1ddZz2p+h/9JbcDs6Kj7A9k3Afh7AH52eR4AvhrAm5Yh9wF4yXJ/5/KMpf+Fy3hNl+ln9tbn2JfbGB/3M13u57tyO8PFxlSYVb/jX+Hsfpj9VJ+TUa2tG+/s6mRVa1/pk7Gqec7Ozv5b7afsxZ6dXTMmZlslL7excQqjWveT6lXJcOvn5HXlVHOZXLburM2t4XlyDXUr958E8P0A/nx5fgqAT805P7s8PwzgGcv9MwA8BABL/6eX8S3qnkuxeex1y53RKd75qCBf2RFBB6s701U6dTDlo5P4useOLvIrXvd1mR1DqWON/JqpjhY6lI9mqrnqeEi1xeeM1x39qbkRszo77Rx7qKMyddRQHQ9Vvs/2ADuGUmvAbKCOS5j8OC+3Kz7MN9xeybopPlEfNo7pl3VixzJqLZU9t+yTTJ3fUP16AI/OOc9WY7fQGOPuMcaZMcYZfOKxdhc0VTvbiDEQdQzEAja7r8atzy4wsoDrdGWBk+GuzkMjPxeUY3vnXNLZgiWE6OQ5qDHcbjPGce7MMvNV9ywJZl2dfTsYFSaXbNZnF7DzerAEohKGwpT9NxMLYrGd2ZP1ueTlkpIqttT+Yf0uoDu5yifdvuhiUWup5jHqfFvmKwB8wxjjxQCeBOAvA3gVgOvHGNct1flNAB5Zxj8C4GYAD48xrgPwZAB/kJnOOe8FcC+Aw8/sJeWUI6mKh811wVORquxcJaDmb+W9lUeFx8lwPLKD5faVXCKoNggbp7BXc7bOd2unAiML6ApbNU4lWjfX+TSbrxJVhUPtGZfAFWXMLNEzfkxWDvQdGzisVTJwz4qv0i/O6do326xaH0Zl5T7nfOWc86Y55y0AXgbgXXPObwbwbgAvXYbdBeAty/39yzOW/nfNDf+lVLWp87hcNWXFtwROJSs+5+pQjYltai6rCLv8VfXI5ipbKb2Z43eDRbUxVT/7KHLYnb0zbrfx1dUlfHXPKu4OXzbWJcOq4DhJslVVamWb7NsxOGV+lT4nKXZWuYynixN536jAXlXgylZx/2XbOX1zseTWe6WL+auQPwDge8cYD+Jwpv6apf01AJ6ytH8vgHu2MN2ysbNy3YqIyWHZWvGNbW6OCqRbqiDGJzpddhaFIW4slzTUfIW/0jVSNwCphOeSRdSPyYvBRenTrZBcAFdBLY5hyZbhUfo4W+cAxvRngUIRszULQm7fKpu7RKXmZtk5YTg+2X/yWnSSdOTFAjQjFSOqZxXsq5jx2Lhj+PMDt405z55vdKZApWQOvCxg5z6VsbvPbL6TxTa8m1thZfwzLnUf52diY93cjl2VfqqPBWRmx45+1fpk3mxexx5unpurkkqFUwWnDi4lQ/leZ11zn7NhN5EybFvWOY7bsocqmWxdHXV8jmFx8QMDR/7nB3C+cs5QzNHUHFXhqIzYqRi6z4ofk13N3TKezVNV7UosMHT4xOpri+0iv2r8SSqdrRiiDs4PnY+5tpOOye0MZ3ffqP4cMBQuJo/xz77gEkMVTJ1+yqYdfTpr7WR2k7rDF+c72dl2zI8VHU1wX8kZyFVRcYzLkJVxWKCr+Fa4GXXHuypQYXbVoAuqFQaVEFyiddUiG69wZcxVgq/WMG8WlrgyDxWM2HhXGTKsaqN311utpfJPVjm6dVbrzWwiq0xhi8rHK31zMFdrm8erfmUPNo61Z13VGLf/nJzzMGkWx/dLTIwqx1XPikcVbOK4nP1Z1VrJZlQFWLfwCtPaV81llTazpaqwqopjS3WxYokfhaPi2+mv7NOdzzYqqxRZJav8r7MukQ9rY+sWg53yX2WbqmJUQTHP7QRnpVdlByfHjc/tlexOQaawbC2qMp6TVO9HG9zZBqrGZWNmR2ZVNwvgKqA4Z9mSlfN8RayiYTpEzKoqUo6R9Wbyc2BXFVfE4fRTFWKcr7CwxMKwqbVSspiOKomxIBpxZsyMsn9G3qo9Xlllp6rZjmyXLDKGzJfZP/Nh2HOCibyVTbKeGbtai8yH+Vd+Zr4RcVd+nv2MjVN+n3E6uyo6umMZZUyAB7tMLAB2qg5nxG7gPmmAz/Kdjp3Kamvl7No7FZ0LYCzhuGDgKryqEtqy3kw3lcwYHqVDlawV5fV3gZnN7SYVFdRyMMuysu5MXteXcnsukBgWxp/tGVchb92Pile1jmoNXbyq9hmjOWCPZY4uuKtqVWVhlQTYc5zH5GaqNpza1G5zZZlZhsrszgmq9siH2ShuLCabYcn8GVYXIDvVaie4MPt0bOJ8w+kVKSeBrq+ypKJ0yUEu2yjLYfLynChPBXuntyLFK/Jhe4mta+VjUYZKht14wHi5NobLYWU+GfkzvTp7uqKjC+6Z1MK5KuEk1QeTG/m5Md12V1WwwOgCT7XxWGDdikk5O+ObeTCsrmpjeBR+traVLoqH4tsN1nmOoqrQYDyVD7BxkXcM2spWSi8XYF0AiuQSPuPJ1rvaH85uTL5LdKqgiPwzMd9j/JVMxd8lyHXtu3Qc33Pff2Zvp5122ukE9BfsZ/Zc1bWFqioi4qkwV8Re2939Vr6qwlI2cq+wnWqzW0Gf9JVX8Tp2uhS+ciXoasHZIef7W+ZcLlKyXBwAlmezJ4722zIV5des+PrVfVXu8HZtq4xLTS6w5yOM/CqqXt1UwIz3Sm/2mq+OZSK22B75xI9ao/w67RKQwp5two6XulfFl/Fnr+zsqIWtmVo/pUPVnvF07LBVhjqiUHMyJsaP8Xd26dI6Ph9z5Hvmz9W6dTHlsdHHlc3Y/q30Psrg3lGCOagLFoxy4FJnf9UZ8MWc86r+TrXhKnoWhNm5X3Ycde7KeOZNwuy/dT3YPBXEXYBQvFmQVdTd+Lmw2HKmy/TqvLkq7LGd4WGJMM9xSb6LMc9h/uhk5yIg6sPmXGzAZXiynNi22tXZ2CXt7IdMVqbss9Ua7GfuO+20005XLV1lZ+5AfR62kqsM1OsMI5YVczXKqlMnN98zTG5cxsHI2WuL/llv9azGunlV9Rb74vyTzHFrtlWfjiyFV+ng1qLTz/A4H1FvClt9uiPTvdk5/1VvqGqvMEyV/1f7LOrE+DEeW9eg649b9qwbdpTHMpm6r7LudVn15wVTr18uIFSBjb3SM/wqwGVnzK+dDl8co+ygdFL2y7iznNzHnpmuGVtek7Vt/WQ8LpCxdvbM1l/Nc4FN+VXWVdlW2buiaB+HPc9hejg++bhBrb/zSyY3j3fB2Mlx+zX3K1tnX1OB/yTyq33o1r7rD90fyP7wGOP9Y4xfH2OcWdpuHGO8Y4zxoeV6w9I+xhg/NcZ4cIzxvjHG83pQjHyRDZlBq4/ikRePBf54X2VW5uSdoFptLpZM8kbLGFylEnnGMcxesZ8FuXjtVlS5XQXt3Jf1Vjqq6ool87z+Snd2H5/z3GxPdmXBjfmMWuM43hU1bPza7nyaJS92z3TKvux8lO25yCPzUr7C5uT+vK/jVekfqdrPzmczlpWfKlTUujlqnbmPMT4M4LY55ydD248B+MM554+MMe4BcMOc8weWn+P7pwBeDOD5AF4153x+wX9DbbLTTjvttNNC8sz9Yo5l7gRw33J/H4CXhPbXzgP9Cg6/tfp0y+lWAPPwWS6P3Uz+nNvU2DjeffI4Jiu3MyxVu8Ko9OhirTA4fkzvrm6Kv1qDav22rhvT3/HZYoOuTtXaKmzO9mye09XhcDKq8c42zP4Ms1p7ZV/Hq1qXjo7O1krnLXLUujD7OZmddXHUDe4TwH8ZY5wdY9y9tD1tzvnR5f5jAJ623D8DwENh7sNL23k0xrh7jHFmjHEGnwjtFeIIqnH+dpJzx/WZ8WevfGy8eh3unCFmvF2bVJgVP3Y8UZ2hqjNBdQzD5qujCHUE0D0qqY7iFLFXX3f0V/GIbezopXPkoo4L2Dox33f6M126r/yZlGx21MDsqmzkjuTUEYbb/6p9ix/HfnbcE5/VsQ/Dzo5GGe8udb8t85VzzkfGGF8E4B1jjN+OnXPOufVoZc55L4B7AWDc9thcZwS1AJUR2UZVZ+gsaKtzSRW8FV4mt3uWxnSP2J31ma5Ozzw28lC81LlubmPjnB2ZvdTauoDLgqbzLYch68+eVQBjOuY2tj6OtwvwTFY3wTFfUcWDSvbM7iy5uf3qEmbkGfV38YLJ7iRlNbaKF8omrt3Z+7y5Zi1bwX3O+chyfXSM8WYAXw7g42OMp885P7ocuzy6DH8EwM1h+k1LW4s6ATG3dXhU2V1VDtVcxU+Ny3PURu3MZ7JUtanG5+Cs7KkqJbZB89yT9CnseQNX81m/W2OXpKpklzeyCwKsv9PuAl41P/KoklTWTwXvdY4qNKoiRBUVlR4Ok0rGax9LnE5GhZEVdC7ZqT1a6ZtpDhvb62OZMcbnjTG+YL0H8CIAHwBwP4C7lmF3AXjLcn8/gG9dvjVzO4BPh+ObFmC1GbOBWWBiwZItROa3Pqu2/Kn6u5+OHOYoaqzbnExutmdFrurIcpXsSvfcHu9V5ef4xzH5XvGP99k2OaDnpMPspWymxlQVH0vSav1YQlJtWW+FU9lEBXY2zwU6hqFTtLCxzme7RZzDqPRUuB2PtZ0lIBXHFHUq96cBePMYYx3/C3PO/zzGeC+AN44xXgHgIwC+cRn/Nhy+KfMggM8AeHlDxjliwFVGUxkb4FnbLR6bp8bk4OE2cOSpsrPSL/PNmy/3ZXl5wzH5mW/GpK5KF1flZDl5fZT+HftkG7u1dQGGBfisP/PPbHOmexyv9MqJgtlb+XPWvwoCKqkwPJl/lut0Zkkpj89j2JXxVLzzOipdcjuzTcbq1ibLZ3Zh5LAoG1R0HH9+4LYxcUYHGRVwlXEr5VUgc87UWZjOuC3B0SWFLYHPYXK83OZnY5U+ebwL8i6YrWPd+ipcbnNVQTLLdQWI0reDrbKLCtjKJh3/YvM7ga6zZlWArniqOFDZQa2F8yNlE+c7il9nLuOjYpjSeUwAQ38V8jj+/MBCUbnKMXNbNJALJGub4quqom62rMYpHGpDKTxdeW7DOF55LVzF5ZJsrKA6ctl9rrIqXIp/FQDZ/VY/dEEnY88ylD1Zten0YHizTMZPBV8VaCN/NlcFeBVIc5Bz69splJQNVGJW/dX8zCvvNVcoKDlsHN2DGspxBXfAL5SrFl3f1sqvm3U7TrWVsj6u8nEVGHMuNieOVQ7JnDPj7eiyEuOT+1mSyLiyrM4m6gYwp0/WI89XPqSCd8ePO5XcSf3R+fuW/cAqTxWcnU22rk1OHMx/FK5qXAdHZ322FlcZazc5RDrKvy3jMuuWuS4Du/nOQVilsxVblbVZFRPbqw3GKkI1Nt6zCi9vmK7Onc24jqsClEomLOF1gl01L8vKeuf1UYGdkas2md+rpFrpwRJ7lsNstSXAxI/SYR3rAm58zjxVglYBlVH2H+bHLuaoPcD2Wp7fkVURS4AdOrrgzjaRWzQ2pgos+ZkFjyqwqza2uVS1EolVb1WVUFUBzHYK48rbVVCOOoE/YmJBS214dp+DpJKvkjGzt0uGLKDndoWTBTYmo1O9VsnKVd4Rn/NJF5gyD7UnXCB3uivdOpU4q5QZP7VnIxa1f7uFmfLXKgm49evY8fz+Y/gH1TH+BMADp41jIz0VwCfLUcdDVxteYMd8JehqwwvsmCP9tTnnF7KOYzlzf0D9i++x0hjjzNWE+WrDC+yYrwRdbXiBHXOXju5YZqeddtppp4unPbjvtNNOO12DdCzB/d7TBnACutowX214gR3zlaCrDS+wY27RUfyD6k477bTTTpeWjqVy32mnnXba6RLSHtx32mmnna5BOvXgPsa4Y4zxwPKD2vecNh4AGGPcPMZ49xjjt8YYvznG+O6l/Yr9KPhFYH/cGOPXxhhvXZ6fOcZ4z4LtDWOMJyztT1yeH1z6bzkFrNePMd40xvjtMcYHxxgvOHYbjzH+2eITHxhjvH6M8aRjs/EY4+fGGI+OMT4Q2jbbdYxx1zL+Q2OMu04B879afON9Y4w3jzGuD32vXDA/MMb42tB+ReIJwxv6vm+MMccYT12eT8fGc85T+wB4HIDfBfAsAE8A8BsAnnOamBZcTwfwvOX+CwD8DoDnAPgxAPcs7fcA+NHl/sUA/hOAAeB2AO85RezfC+AXALx1eX4jgJct968G8B3L/XcCePVy/zIAbzgFrPcB+Pbl/gkArj9mG+Pwc5G/D+Bzgm2/7dhsDODvAHgegA+Etk12BXAjgN9brjcs9zdcYcwvAnDdcv+jAfNzlljxRADPXGLI465kPGF4l/abAbwdhz+D/tTTtPEV3RzEQC8A8Pbw/EoArzxNTALnWwB8DQ7/Fe3Tl7an4/AfXwHAzwD4pjD+3LgrjPMmAO8E8NUA3ro40yfDBjln78UBX7DcX7eMG1cQ65OXQDlS+9HaGI/9PvCNi83eCuBrj9HGAG5JgXKTXQF8E4CfCe3njbsSmFPf3wfwuuX+vDix2vlKxxOGF8CbAPxtAB/GY8H9VGx82scyrR/TPk1aXqWfC+A9uMgfBb8C9JMAvh/Any/PTwHwqTnnZwmuc5iX/k8v468UPRPAJwD8u+UY6WfH4Ze+jtbG8/Bzk/8awP8C8FEcbHYWx2vjSFvteur2TvSPcah+gSPFPMa4E8Ajc87fSF2ngve0g/tR0xjj8wH8EoDvmXP+ceybh1R7NN8jHWN8PYBH55xnTxtLk67D4bX2p+eczwXwf3A4LjhHR2jjGwDciUNi+qsAPg/AHacK6gR0bHataIzxQwA+C+B1p41F0RjjcwH8IIB/ftpYVjrt4H5RP6Z9OWmM8XgcAvvr5py/vDR/fBx+DBzjEv4o+CWirwDwDWOMDwP4RRyOZl4F4Poxxvo3hCKuc5iX/icD+IMriPdhAA/POd+zPL8Jh2B/zDb+uwB+f875iTnnnwH4ZRzsfqw2jrTVrsdgb4wxvg3A1wP45iUpAceJ+a/jkPR/Y9mDNwH41THGXzG4Live0w7u7wXw7OXbBk/A4R+d7j9lTBhjDACvAfDBOeePh67L8qPgl4LmnK+cc94057wFBzu+a875zQDeDeClAvOqy0uX8VesmptzfgzAQ2OML1maXgjgt3DENsbhOOb2McbnLj6yYj5KGyfaate3A3jRGOOG5Y3lRUvbFaMxxh04HDN+w5zzM6HrfgAvW76N9EwAzwbwP3GK8WTO+f455xfNOW9Z9uDDOHwp42M4LRtfzn8gaf6jxItx+DbK7wL4odPGs2D6ShxeW98H4NeXz4txOC99J4APAfivAG5cxg8A/3bR4f0Abjtl/F+Fx74t8ywcHP9BAP8BwBOX9ictzw8u/c86BZxfBuDMYuf/EhG9wgAAAKlJREFUiMM3Bo7axgD+BYDfBvABAD+Pwzc2jsrGAF6Pw78J/BkOQeYVJ7ErDufcDy6fl58C5gdxOJNe9+Crw/gfWjA/AODrQvsViScMb+r/MB77B9VTsfH+5wd22mmnna5BOu1jmZ122mmnnS4D7cF9p5122ukapD2477TTTjtdg7QH95122mmna5D24L7TTjvtdA3SHtx32mmnna5B2oP7TjvttNM1SP8foMKhg6UQv0IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "output_file_name = 'adv_signed_gradient_overall.png'\n",
    "\n",
    "save_color_img_to_png(overall_image, output_file_name)\n",
    "show_png(output_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [0.9987436]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]], shape=(32, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "y_pred_tensor = tf.convert_to_tensor(model.predict(x_tensor.numpy()), dtype=tf.float32)\n",
    "print(y_pred_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3.5762787e-07]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [8.9406967e-08]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [2.7120113e-06]\n",
      " [0.0000000e+00]\n",
      " [8.1419945e-05]\n",
      " [0.0000000e+00]\n",
      " [1.1920929e-07]\n",
      " [0.0000000e+00]\n",
      " [3.5226345e-05]\n",
      " [0.0000000e+00]\n",
      " [5.9235096e-04]\n",
      " [1.8671155e-04]\n",
      " [2.2042274e-02]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [9.2387199e-07]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]\n",
      " [0.0000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "adv_noise_units = tf.constant(1/255.0 * 12.0, dtype=tf.float32)\n",
    "x_adv_tensor = x_tensor - adv_noise_units * tf.cast(adv_sign_grad_tensor, dtype=tf.float32)\n",
    "y_adv_pred_tensor = model.predict(x_adv_tensor.numpy())\n",
    "print(y_adv_pred_tensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ricky's Comment\n",
    "\n",
    "- To visualize the sign gradient of 32 samples, we plot the graph in RGB. Red is 1, green is -1, blue is 0, black is a boderline to separate sample.\n",
    "- There is no change in the last packet i.e. the 16th packet.\n",
    "- From my eyeballs, there is a common pattern among 32 samples. Would it be possible to find a universal adversarial noise to apply to all input X?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deepfool Approach\n",
    "\n",
    "- Deepfool. See [paper](https://arxiv.org/abs/1511.04599)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model(x) -> y , where y in [0, 1]\n",
    "# f(model(x)) = 2 * model(x) - 1, f(x) in [-1, 1]\n",
    "def map_func(x):\n",
    "    return tf.constant(2.0, dtype=tf.float32) * x - tf.constant(1.0, dtype=tf.float32)\n",
    "\n",
    "@tf.function\n",
    "def model_gradient(x_in, trained_model=model):\n",
    "    x = tf.cast(x_in, tf.float32)\n",
    "    with tf.GradientTape() as tape:\n",
    "        tape.watch(x)\n",
    "        y_pred = trained_model(x)\n",
    "        y_map = map_func(y_pred)\n",
    "    gradient = tape.gradient(y_map, x)\n",
    "    return gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_tensor:  tf.Tensor(\n",
      "[[[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01960784 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.         ... 0.         0.         0.        ]]\n",
      "\n",
      " [[0.27058825 0.         0.00784314 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  ...\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.00392157 ... 0.         0.         0.        ]\n",
      "  [0.27058825 0.         0.01176471 ... 0.         0.         0.        ]]], shape=(32, 16, 1500), dtype=float32)\n",
      "gradient_tensor:  tf.Tensor(\n",
      "[[[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  ...\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]\n",
      "  [0. 0. 0. ... 0. 0. 0.]]], shape=(32, 16, 1500), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# batch version\n",
    "x_tensor = tf.convert_to_tensor(x, dtype=tf.float32)\n",
    "gradient_tensor = model_gradient(x_tensor)\n",
    "print('x_tensor: ', x_tensor)\n",
    "print('gradient_tensor: ', gradient_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [0.9987436]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]\n",
      " [1.       ]]\n"
     ]
    }
   ],
   "source": [
    "print(model.predict(x_tensor.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decimal: 20: False\n",
      "decimal: 19: False\n",
      "decimal: 18: False\n",
      "decimal: 17: False\n",
      "decimal: 16: False\n",
      "decimal: 15: False\n",
      "decimal: 14: False\n",
      "decimal: 13: False\n",
      "decimal: 12: False\n",
      "decimal: 11: False\n",
      "decimal: 10: False\n",
      "decimal: 9: False\n",
      "decimal: 8: False\n",
      "decimal: 7: False\n",
      "decimal: 6: False\n",
      "decimal: 5: False\n",
      "decimal: 4: False\n",
      "decimal: 3: False\n",
      "decimal: 2: True\n",
      "decimal: 1: True\n"
     ]
    }
   ],
   "source": [
    "def almost_equals(a, b, decimal=6):\n",
    "    try:\n",
    "        np.testing.assert_almost_equal(a, b, decimal=decimal)\n",
    "    except AssertionError:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "def detect_zeros(input_tensor, num_decimal=20):\n",
    "    # comparing to zeros:\n",
    "    zeros_like = np.zeros_like(input_tensor.numpy())  \n",
    "    for decimal in range(num_decimal, 0, -1):\n",
    "        print('decimal: {0}: {1}'.format(decimal,\n",
    "                                         almost_equals(zeros_like,\n",
    "                                                       input_tensor.numpy(),\n",
    "                                                       decimal=decimal)))\n",
    "detect_zeros(gradient_tensor, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ricky's Comment\n",
    "\n",
    "- The forward gradient looks like a zero tensor. It is caused by the vanishing gradient in the sigmoid activation function of the output layer. The sigmoid function satuarates at both ends.\n",
    "- There is numerical difficulty in DeepFool and JASM approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}