DA_GAN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DA-GAN",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "RTDZpESsjdwB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q vaegan.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hzYbEqYj69i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os.path as osp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-KXKsXzkEIM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "2289ba88-ffbe-48d4-9115-2c35e8cfa41f"
      },
      "source": [
        "# Set Gogle Drive Connection\n",
        "if not osp.exists(\"./drive\"):\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HXft0JsTkKag",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "outputId": "185312de-d5f8-4f12-a4cc-3dc204dccfec"
      },
      "source": [
        "torchvision_version = !python -c \"import torchvision; print(torchvision.__version__)\"\n",
        "if torchvision_version[0] != '0.4.0':\n",
        "  print(\"Incorrect torchvision version is installed. Uninstalling.\")\n",
        "  # Uninstall old torchvision\n",
        "  !pip uninstall -y torchvision\n",
        "  print(\"Installing torchvision 0.4.0\")\n",
        "  # Install proper torchvision\n",
        "  !pip install torchvision==0.4.0 --upgrade"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Incorrect torchvision version is installed. Uninstalling.\n",
            "Uninstalling torchvision-0.3.0:\n",
            "  Successfully uninstalled torchvision-0.3.0\n",
            "Installing torchvision 0.4.0\n",
            "Collecting torchvision==0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/06/e6/a564eba563f7ff53aa7318ff6aaa5bd8385cbda39ed55ba471e95af27d19/torchvision-0.4.0-cp36-cp36m-manylinux1_x86_64.whl (8.8MB)\n",
            "\u001b[K     |████████████████████████████████| 8.8MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from torchvision==0.4.0) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision==0.4.0) (4.3.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision==0.4.0) (1.16.4)\n",
            "Collecting torch==1.2.0 (from torchvision==0.4.0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/30/57/d5cceb0799c06733eefce80c395459f28970ebb9e896846ce96ab579a3f1/torch-1.2.0-cp36-cp36m-manylinux1_x86_64.whl (748.8MB)\n",
            "\u001b[K     |████████████████████████████████| 748.9MB 18kB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision==0.4.0) (0.46)\n",
            "Installing collected packages: torch, torchvision\n",
            "  Found existing installation: torch 1.1.0\n",
            "    Uninstalling torch-1.1.0:\n",
            "      Successfully uninstalled torch-1.1.0\n",
            "Successfully installed torch-1.2.0 torchvision-0.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H_Z5OxyqkMbT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        },
        "outputId": "9ba56812-0073-43e1-e10f-ecabf34283ad"
      },
      "source": [
        "if not osp.exists(\"apex\"):\n",
        "  print(\"Cloning NVIDIA APEX\")\n",
        "  !git clone https://github.com/NVIDIA/apex"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning NVIDIA APEX\n",
            "Cloning into 'apex'...\n",
            "remote: Enumerating objects: 7, done.\u001b[K\n",
            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
            "remote: Total 5076 (delta 2), reused 2 (delta 0), pack-reused 5069\u001b[K\n",
            "Receiving objects: 100% (5076/5076), 8.82 MiB | 5.16 MiB/s, done.\n",
            "Resolving deltas: 100% (3275/3275), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3KJvT4HnkNsI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bcf7e6e3-417f-4e96-f604-7760a4805180"
      },
      "source": [
        "vis_inst = !python -c 'import pkgutil; print(1 if pkgutil.find_loader(\"apex\") else 0)'\n",
        "if not int(vis_inst[0]):\n",
        "  print(\"Apex not installed\")\n",
        "  !cd apex && pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./\n",
        "else:\n",
        "  print(\"Apex is already installed\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Apex not installed\n",
            "/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py:243: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\n",
            "  cmdoptions.check_install_build_global(options)\n",
            "Created temporary directory: /tmp/pip-ephem-wheel-cache-v6__vp3b\n",
            "Created temporary directory: /tmp/pip-req-tracker-gtnh1zou\n",
            "Created requirements tracker '/tmp/pip-req-tracker-gtnh1zou'\n",
            "Created temporary directory: /tmp/pip-install-0gdwylfb\n",
            "Processing /content/apex\n",
            "  Created temporary directory: /tmp/pip-req-build-33_wi562\n",
            "  Added file:///content/apex to build tracker '/tmp/pip-req-tracker-gtnh1zou'\n",
            "    Running setup.py (path:/tmp/pip-req-build-33_wi562/setup.py) egg_info for package from file:///content/apex\n",
            "    Running command python setup.py egg_info\n",
            "    torch.__version__  =  1.2.0\n",
            "    running egg_info\n",
            "    creating pip-egg-info/apex.egg-info\n",
            "    writing pip-egg-info/apex.egg-info/PKG-INFO\n",
            "    writing dependency_links to pip-egg-info/apex.egg-info/dependency_links.txt\n",
            "    writing top-level names to pip-egg-info/apex.egg-info/top_level.txt\n",
            "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\n",
            "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\n",
            "    /tmp/pip-req-build-33_wi562/setup.py:33: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "      warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "  Source in /tmp/pip-req-build-33_wi562 has version 0.1, which satisfies requirement apex==0.1 from file:///content/apex\n",
            "  Removed apex==0.1 from file:///content/apex from build tracker '/tmp/pip-req-tracker-gtnh1zou'\n",
            "Skipping bdist_wheel for apex, due to binaries being disabled for it.\n",
            "Installing collected packages: apex\n",
            "  Created temporary directory: /tmp/pip-record-stemxx4p\n",
            "    Running command /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-req-build-33_wi562/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-req-build-33_wi562/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-stemxx4p/install-record.txt --single-version-externally-managed --compile\n",
            "    torch.__version__  =  1.2.0\n",
            "    /tmp/pip-req-build-33_wi562/setup.py:33: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "      warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "\n",
            "    Compiling cuda extensions with\n",
            "    nvcc: NVIDIA (R) Cuda compiler driver\n",
            "    Copyright (c) 2005-2018 NVIDIA Corporation\n",
            "    Built on Sat_Aug_25_21:08:01_CDT_2018\n",
            "    Cuda compilation tools, release 10.0, V10.0.130\n",
            "    from /usr/local/cuda/bin\n",
            "\n",
            "    running install\n",
            "    running build\n",
            "    running build_py\n",
            "    creating build\n",
            "    creating build/lib.linux-x86_64-3.6\n",
            "    creating build/lib.linux-x86_64-3.6/apex\n",
            "    copying apex/__init__.py -> build/lib.linux-x86_64-3.6/apex\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof\n",
            "    copying apex/pyprof/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof\n",
            "    creating build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/LARC.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/__init__.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/multiproc.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/distributed.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/optimized_sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    creating build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/__init__.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/models.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/RNNBackend.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/cells.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    creating build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    copying apex/normalization/__init__.py -> build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    copying apex/normalization/fused_layer_norm.py -> build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    creating build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    creating build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/fp16util.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/loss_scaler.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/__init__.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    creating build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/__init__.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/weight_norm.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/reparameterization.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    creating build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/__version__.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/rnn_compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_amp_state.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/opt.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/wrap.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/utils.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_initialize.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_process_optimizer.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/frontend.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/amp.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/handle.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/scaler.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    creating build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    copying apex/multi_tensor_apply/__init__.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/nvvp.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/__main__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/kernel.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/db.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/parse.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    copying apex/pyprof/nvtx/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    copying apex/pyprof/nvtx/nvmarker.py -> build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/pointwise.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/blas.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/utility.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/pooling.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/softmax.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/linear.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/base.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/recurrentCell.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/usage.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/dropout.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/activation.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/loss.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/misc.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/prof.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/reduction.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/data.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/output.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/normalization.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/__main__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/convert.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/randomSample.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/embedding.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/index_slice_join_mutate.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/conv.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/optim.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    creating build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/functional_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/torch_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/tensor_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    running build_ext\n",
            "    building 'apex_C' extension\n",
            "    creating build/temp.linux-x86_64-3.6\n",
            "    creating build/temp.linux-x86_64-3.6/csrc\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/include/python3.6m -c csrc/flatten_unflatten.cpp -o build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=apex_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -o build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'amp_C' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/amp_C_frontend.cpp -o build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_scale_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_axpby_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_l2norm_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_1.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_2.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'fused_adam_cuda' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/fused_adam_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/fused_adam_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'syncbn' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/syncbn.cpp -o build/temp.linux-x86_64-3.6/csrc/syncbn.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/welford.cu -o build/temp.linux-x86_64-3.6/csrc/welford.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/syncbn.o build/temp.linux-x86_64-3.6/csrc/welford.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'fused_layer_norm_cuda' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o -O3 -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -maxrregcount=50 -O3 --use_fast_math -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\n",
            "    running install_lib\n",
            "    copying build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/nvvp.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/__main__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/db.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/parse.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/nvtx/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/nvtx/nvmarker.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/pointwise.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/blas.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/utility.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/pooling.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/softmax.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/linear.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/base.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/recurrentCell.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/usage.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/dropout.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/activation.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/loss.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/misc.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/prof.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/reduction.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/data.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/output.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/normalization.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/__main__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/convert.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/randomSample.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/embedding.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/index_slice_join_mutate.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/conv.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/optim.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/LARC.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm_kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/multiproc.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm_kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/distributed.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/models.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/RNNBackend.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/cells.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/normalization/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/normalization/fused_layer_norm.py -> /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_adam.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fp16_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16util.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/loss_scaler.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/weight_norm.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/reparameterization.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/__version__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/rnn_compat.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/compat.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_amp_state.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/opt.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/wrap.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/utils.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_initialize.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/functional_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/torch_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/tensor_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_process_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/frontend.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/amp.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/handle.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/scaler.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/multi_tensor_apply.py -> /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    copying build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    copying build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    copying build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/nvvp.py to nvvp.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/__main__.py to __main__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/kernel.py to kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/db.py to db.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/parse.py to parse.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx/nvmarker.py to nvmarker.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/pointwise.py to pointwise.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/blas.py to blas.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/utility.py to utility.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/pooling.py to pooling.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/softmax.py to softmax.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/linear.py to linear.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/base.py to base.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/recurrentCell.py to recurrentCell.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/usage.py to usage.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/dropout.py to dropout.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/activation.py to activation.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/loss.py to loss.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/misc.py to misc.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/prof.py to prof.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/reduction.py to reduction.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/data.py to data.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/output.py to output.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/normalization.py to normalization.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/__main__.py to __main__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/convert.py to convert.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/randomSample.py to randomSample.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/embedding.py to embedding.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/index_slice_join_mutate.py to index_slice_join_mutate.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/conv.py to conv.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/optim.py to optim.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/LARC.py to LARC.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/distributed.py to distributed.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/models.py to models.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/cells.py to cells.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/normalization/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/__version__.py to __version__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/compat.py to compat.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/opt.py to opt.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/wrap.py to wrap.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/utils.py to utils.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_initialize.py to _initialize.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/frontend.py to frontend.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/amp.py to amp.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/handle.py to handle.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/scaler.py to scaler.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\n",
            "    running install_egg_info\n",
            "    running egg_info\n",
            "    creating apex.egg-info\n",
            "    writing apex.egg-info/PKG-INFO\n",
            "    writing dependency_links to apex.egg-info/dependency_links.txt\n",
            "    writing top-level names to apex.egg-info/top_level.txt\n",
            "    writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "    writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "    Copying apex.egg-info to /usr/local/lib/python3.6/dist-packages/apex-0.1-py3.6.egg-info\n",
            "    running install_scripts\n",
            "    writing list of installed files to '/tmp/pip-record-stemxx4p/install-record.txt'\n",
            "  Running setup.py install for apex ... \u001b[?25l\u001b[?25hdone\n",
            "  Removing source in /tmp/pip-req-build-33_wi562\n",
            "Successfully installed apex-0.1\n",
            "Cleaning up...\n",
            "Removed build tracker '/tmp/pip-req-tracker-gtnh1zou'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rs8bZDuylnf0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205
        },
        "outputId": "d55f9a5c-90b6-4929-8ce8-b9ac94ec0f86"
      },
      "source": [
        "!pip install kaptan==0.5.12"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting kaptan==0.5.12\n",
            "  Downloading https://files.pythonhosted.org/packages/94/64/f492edfcac55d4748014b5c9f9a90497325df7d97a678c5d56443f881b7a/kaptan-0.5.12.tar.gz\n",
            "Requirement already satisfied: PyYAML<6,>=3.13 in /usr/local/lib/python3.6/dist-packages (from kaptan==0.5.12) (3.13)\n",
            "Building wheels for collected packages: kaptan\n",
            "  Building wheel for kaptan (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaptan: filename=kaptan-0.5.12-cp36-none-any.whl size=10976 sha256=b35a4ae75b21d9377cd486bd396c05e460d2da130ece9919438bf6afd220c0fa\n",
            "  Stored in directory: /root/.cache/pip/wheels/27/7c/59/f347d4c2e1504da5cd2eb40d07cbcb341aefbdb1682d83a4aa\n",
            "Successfully built kaptan\n",
            "Installing collected packages: kaptan\n",
            "Successfully installed kaptan-0.5.12\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j894dpKWkY4I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os, shutil, glob, re, pdb, json\n",
        "import kaptan\n",
        "#import click\n",
        "import utils\n",
        "import torch, torchsummary, torchvision"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5k_r3vf_kOb8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def main(config, mode, weights):\n",
        "    # Generate configuration\n",
        "    cfg = kaptan.Kaptan(handler='yaml')\n",
        "    config = cfg.import_config(config)\n",
        "\n",
        "    # Generate logger\n",
        "    MODEL_SAVE_NAME, MODEL_SAVE_FOLDER, LOGGER_SAVE_NAME, CHECKPOINT_DIRECTORY = utils.generate_save_names(config)\n",
        "    logger = utils.generate_logger(MODEL_SAVE_FOLDER, LOGGER_SAVE_NAME)\n",
        "\n",
        "    logger.info(\"*\"*40);logger.info(\"\");logger.info(\"\")\n",
        "    logger.info(\"Using the following configuration:\")\n",
        "    logger.info(config.export(\"yaml\", indent=4))\n",
        "    logger.info(\"\");logger.info(\"\");logger.info(\"*\"*40)\n",
        "\n",
        "    \"\"\" SETUP IMPORTS \"\"\"\n",
        "    #from crawlers import ReidDataCrawler\n",
        "    #from generators import SequencedGenerator\n",
        "    \n",
        "\n",
        "    #from loss import LossBuilder\n",
        "    \n",
        "\n",
        "    NORMALIZATION_MEAN, NORMALIZATION_STD, RANDOM_ERASE_VALUE = utils.fix_generator_arguments(config)\n",
        "    TRAINDATA_KWARGS = {\"rea_value\": config.get(\"TRANSFORMATION.RANDOM_ERASE_VALUE\")}\n",
        "\n",
        "\n",
        "    \"\"\" Load previousely saved logger, if it exists \"\"\"\n",
        "    DRIVE_BACKUP = config.get(\"SAVE.DRIVE_BACKUP\")\n",
        "    if DRIVE_BACKUP:\n",
        "        backup_logger = os.path.join(CHECKPOINT_DIRECTORY, LOGGER_SAVE_NAME)\n",
        "        if os.path.exists(backup_logger):\n",
        "            shutil.copy2(backup_logger, \".\")\n",
        "    else:\n",
        "        backup_logger = None\n",
        "\n",
        "    NUM_GPUS = torch.cuda.device_count()\n",
        "    if NUM_GPUS > 1:\n",
        "        raise RuntimeError(\"Not built for multi-GPU. Please start with single-GPU.\")\n",
        "    logger.info(\"Found %i GPUs\"%NUM_GPUS)\n",
        "\n",
        "\n",
        "    # --------------------- BUILD GENERATORS ------------------------\n",
        "    # Supported integrated data sources --> MNIST, CIFAR\n",
        "    # For BDD or others need a crawler and stuff...but we;ll deal with it later\n",
        "    from generators import ClassedGenerator\n",
        "\n",
        "    load_dataset = config.get(\"EXECUTION.DATASET_PRELOAD\")\n",
        "    if load_dataset in [\"MNIST\", \"CIFAR10\", \"CIFAR100\"]:\n",
        "        crawler = load_dataset\n",
        "        #dataset = torchvision.datasets.MNIST(root=\"./MNIST\", train=True,)\n",
        "        logger.info(\"No crawler necessary when using %s dataset\"%crawler)\n",
        "    else:\n",
        "        raise NotImplementedError()\n",
        "        # import crawler\n",
        "        # crawler = crawler...\n",
        "        # logger.info(\"Crawling data folder %s\"%config.get(\"DATASET.ROOT_DATA_FOLDER\"))\n",
        "        # crawler = ReidDataCrawler(data_folder = config.get(\"DATASET.ROOT_DATA_FOLDER\"), train_folder=config.get(\"DATASET.TRAIN_FOLDER\"), test_folder = config.get(\"DATASET.TEST_FOLDER\"), query_folder=config.get(\"DATASET.QUERY_FOLDER\"), **{\"logger\":logger})\n",
        "\n",
        "    train_generator = ClassedGenerator( gpus=NUM_GPUS, i_shape=config.get(\"DATASET.SHAPE\"), \\\n",
        "                                        normalization_mean=NORMALIZATION_MEAN, normalization_std=NORMALIZATION_STD, normalization_scale=1./config.get(\"TRANSFORMATION.NORMALIZATION_SCALE\"), \\\n",
        "                                        h_flip = config.get(\"TRANSFORMATION.H_FLIP\"), t_crop=config.get(\"TRANSFORMATION.T_CROP\"), rea=config.get(\"TRANSFORMATION.RANDOM_ERASE\"), \n",
        "                                        **TRAINDATA_KWARGS)    \n",
        "    train_generator.setup(  crawler, preload_classes = config.get(\"EXECUTION.DATASET_PRELOAD_CLASS\"), \\\n",
        "                            mode='train',batch_size=config.get(\"TRANSFORMATION.BATCH_SIZE\"), \\\n",
        "                            workers = config.get(\"TRANSFORMATION.WORKERS\"))\n",
        "    logger.info(\"Generated training data generator\")\n",
        "\n",
        "    test_generator = ClassedGenerator( gpus=NUM_GPUS, i_shape=config.get(\"DATASET.SHAPE\"), \\\n",
        "                                        normalization_mean=NORMALIZATION_MEAN, normalization_std=NORMALIZATION_STD, normalization_scale=1./config.get(\"TRANSFORMATION.NORMALIZATION_SCALE\"), \\\n",
        "                                        h_flip = config.get(\"TRANSFORMATION.H_FLIP\"), t_crop=config.get(\"TRANSFORMATION.T_CROP\"), rea=config.get(\"TRANSFORMATION.RANDOM_ERASE\"), \n",
        "                                        **TRAINDATA_KWARGS)    \n",
        "    test_generator.setup(  crawler, preload_classes = config.get(\"EXECUTION.DATASET_TEST_PRELOAD_CLASS\"), \\\n",
        "                            mode='test',batch_size=config.get(\"TRANSFORMATION.BATCH_SIZE\"), \\\n",
        "                            workers = config.get(\"TRANSFORMATION.WORKERS\"))\n",
        "    logger.info(\"Generated testing data generator\")\n",
        "\n",
        "\n",
        "    # --------------------- INSTANTIATE MODEL ------------------------\n",
        "    model_builder = __import__(\"models\", fromlist=[\"*\"])\n",
        "    model_builder = getattr(model_builder, config.get(\"EXECUTION.MODEL_BUILDER\"))\n",
        "    logger.info(\"Loaded {} from {} to build VAEGAN model\".format(config.get(\"EXECUTION.MODEL_BUILDER\"), \"models\"))\n",
        "\n",
        "    vaegan_model = model_builder(   arch=config.get(\"MODEL.ARCH\"), base=config.get(\"MODEL.BASE\"), \\\n",
        "                                    latent_dimensions = config.get(\"MODEL.LATENT_DIMENSIONS\"), \\\n",
        "                                    **json.loads(config.get(\"MODEL.MODEL_KWARGS\")))\n",
        "    logger.info(\"Finished instantiating model\")\n",
        "\n",
        "    if mode == \"test\":\n",
        "        vaegan_model.load_state_dict(torch.load(weights))\n",
        "        vaegan_model.cuda()\n",
        "        vaegan_model.eval()\n",
        "    else:\n",
        "        vaegan_model.cuda()\n",
        "        #logger.info(torchsummary.summary(vaegan_model, input_size=(config.get(\"TRANSFORMATION.CHANNELS\"), *config.get(\"DATASET.SHAPE\"))))\n",
        "        logger.info(torchsummary.summary(vaegan_model.Encoder, input_size=(config.get(\"TRANSFORMATION.CHANNELS\"), *config.get(\"DATASET.SHAPE\"))))\n",
        "        logger.info(torchsummary.summary(vaegan_model.Decoder, input_size=(config.get(\"MODEL.LATENT_DIMENSIONS\"), 1)))\n",
        "        logger.info(torchsummary.summary(vaegan_model.LatentDiscriminator, input_size=(config.get(\"MODEL.LATENT_DIMENSIONS\"), 1)))\n",
        "        logger.info(torchsummary.summary(vaegan_model.Discriminator, input_size=(config.get(\"TRANSFORMATION.CHANNELS\"), *config.get(\"DATASET.SHAPE\"))))\n",
        "    \n",
        "\n",
        "    # --------------------- INSTANTIATE LOSS ------------------------\n",
        "    # ----------- NOT NEEDED. VAEGAN WILL USE BCE LOSS THROUGHOUT \n",
        "    # loss_function = LossBuilder(loss_functions=config.get(\"LOSS.LOSSES\"), loss_lambda=config.get(\"LOSS.LOSS_LAMBDAS\"), loss_kwargs=config.get(\"LOSS.LOSS_KWARGS\"), **{\"logger\":logger})\n",
        "    # logger.info(\"Built loss function\")\n",
        "    # --------------------- INSTANTIATE OPTIMIZER ------------------------\n",
        "    optimizer_builder = __import__(\"optimizer\", fromlist=[\"*\"])\n",
        "    optimizer_builder = getattr(optimizer_builder, config.get(\"EXECUTION.OPTIMIZER_BUILDER\"))\n",
        "    logger.info(\"Loaded {} from {} to build VAEGAN model\".format(config.get(\"EXECUTION.OPTIMIZER_BUILDER\"), \"optimizer\"))\n",
        "\n",
        "    OPT = optimizer_builder(base_lr=config.get(\"OPTIMIZER.BASE_LR\"))\n",
        "    optimizer = OPT.build(vaegan_model, config.get(\"OPTIMIZER.OPTIMIZER_NAME\"), **json.loads(config.get(\"OPTIMIZER.OPTIMIZER_KWARGS\")))\n",
        "    logger.info(\"Built optimizer\")\n",
        "    # --------------------- INSTANTIATE SCHEDULER ------------------------\n",
        "    try:\n",
        "        scheduler = __import__('torch.optim.lr_scheduler', fromlist=['lr_scheduler'])\n",
        "        scheduler_ = getattr(scheduler, config.get(\"SCHEDULER.LR_SCHEDULER\"))\n",
        "    except (ModuleNotFoundError, AttributeError):\n",
        "        scheduler_ = config.get(\"SCHEDULER.LR_SCHEDULER\")\n",
        "        scheduler = __import__(\"scheduler.\"+scheduler_, fromlist=[scheduler_])\n",
        "        scheduler_ = getattr(scheduler, scheduler_)\n",
        "    scheduler = {}\n",
        "    for base_model in [\"Encoder\", \"Decoder\", \"Discriminator\", \"Autoencoder\", \"LatentDiscriminator\"]:\n",
        "        scheduler[base_model] = scheduler_(optimizer[base_model], last_epoch = -1, **json.loads(config.get(\"SCHEDULER.LR_KWARGS\")))\n",
        "        logger.info(\"Built scheduler for {}\".format(base_model))\n",
        "    \n",
        "\n",
        "    # --------------------- SETUP CONTINUATION  ------------------------\n",
        "    if DRIVE_BACKUP:\n",
        "        fl_list = glob.glob(os.path.join(CHECKPOINT_DIRECTORY, \"*.pth\"))\n",
        "    else:\n",
        "        fl_list = glob.glob(os.path.join(MODEL_SAVE_FOLDER, \"*.pth\"))\n",
        "    _re = re.compile(r'.*epoch([0-9]+)\\.pth')\n",
        "    previous_stop = [int(item[1]) for item in [_re.search(item) for item in fl_list] if item is not None]\n",
        "    if len(previous_stop) == 0:\n",
        "        previous_stop = 0\n",
        "    else:\n",
        "        previous_stop = max(previous_stop) + 1\n",
        "        logger.info(\"Previous stop detected. Will attempt to resume from epoch %i\"%previous_stop)\n",
        "\n",
        "    # --------------------- INSTANTIATE TRAINER  ------------------------\n",
        "    Trainer = __import__(\"trainer\", fromlist=[\"*\"])\n",
        "    Trainer = getattr(Trainer, config.get(\"EXECUTION.TRAINER\"))\n",
        "    logger.info(\"Loaded {} from {} to build VAEGAN model\".format(config.get(\"EXECUTION.TRAINER\"), \"trainer\"))\n",
        "\n",
        "    loss_stepper = Trainer(model=vaegan_model, loss_fn = None, optimizer = optimizer, scheduler = scheduler, train_loader = train_generator.dataloader, test_loader = test_generator.dataloader, epochs = config.get(\"EXECUTION.EPOCHS\"), batch_size = config.get(\"TRANSFORMATION.BATCH_SIZE\"), latent_size = config.get(\"MODEL.LATENT_DIMENSIONS\"), logger = logger)\n",
        "    loss_stepper.setup(step_verbose = config.get(\"LOGGING.STEP_VERBOSE\"), save_frequency=config.get(\"SAVE.SAVE_FREQUENCY\"), test_frequency = config.get(\"EXECUTION.TEST_FREQUENCY\"), save_directory = MODEL_SAVE_FOLDER, save_backup = DRIVE_BACKUP, backup_directory = CHECKPOINT_DIRECTORY, gpus=NUM_GPUS, fp16 = config.get(\"OPTIMIZER.FP16\"), model_save_name = MODEL_SAVE_NAME, logger_file = LOGGER_SAVE_NAME)\n",
        "    if mode == 'train':\n",
        "      loss_stepper.train(continue_epoch=previous_stop)\n",
        "    elif mode == 'test':\n",
        "      loss_stepper.evaluate()\n",
        "    else:\n",
        "      raise NotImplementedError()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9LgFzLbkcF9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9fefab93-45a5-4107-edcc-f7694d15a379"
      },
      "source": [
        "main(\"config/vaegan_cifar10.yml\", \"train\",\"\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "05:28:44-130 ****************************************\n",
            "05:28:44-132 \n",
            "05:28:44-134 \n",
            "05:28:44-137 Using the following configuration:\n",
            "05:28:44-146 DATASET:\n",
            "    QUERY_FOLDER: ''\n",
            "    ROOT_DATA_FOLDER: CIFAR10\n",
            "    SHAPE: [32, 32]\n",
            "    TEST_FOLDER: ''\n",
            "    TRAIN_FOLDER: ''\n",
            "EXECUTION:\n",
            "    DATASET_PRELOAD: CIFAR10\n",
            "    DATASET_PRELOAD_CLASS: [0, 1, 2]\n",
            "    DATASET_TEST_PRELOAD_CLASS: [3, 4, 5, 6, 7, 8, 9]\n",
            "    EPOCHS: 121\n",
            "    MODEL_BUILDER: vaegan_model_builder\n",
            "    MODEL_SERVING: None\n",
            "    OPTIMIZER_BUILDER: VAEGANOptimizerBuilder\n",
            "    TEST_FREQUENCY: 5\n",
            "    TRAINER: VAEGANTrainer\n",
            "LOGGING: {STEP_VERBOSE: 25}\n",
            "MODEL: {ARCH: VAEGAN, BASE: 32, LATENT_DIMENSIONS: 256, MODEL_KWARGS: '{\"channels\":3}'}\n",
            "OPTIMIZER: {BASE_LR: 0.002, FP16: true, LR_BIAS_FACTOR: 1.0, OPTIMIZER_KWARGS: '{\"betas\":[0.5,\n",
            "        0.999]}', OPTIMIZER_NAME: Adam, WEIGHT_BIAS_FACTOR: 0.0005, WEIGHT_DECAY: 0.0005}\n",
            "SAVE: {DRIVE_BACKUP: true, MODEL_BACKBONE: vaegan, MODEL_CORE_NAME: vaegan_cifar,\n",
            "    MODEL_QUALIFIER: cifar10, MODEL_VERSION: 4, SAVE_FREQUENCY: 5}\n",
            "SCHEDULER: {LR_KWARGS: '{\"step_size\":30, \"gamma\":0.25}', LR_SCHEDULER: StepLR}\n",
            "TRANSFORMATION: {BATCH_SIZE: 64, CHANNELS: 3, H_FLIP: 0, NORMALIZATION_MEAN: 0.5,\n",
            "    NORMALIZATION_SCALE: 255.0, NORMALIZATION_STD: 0.5, RANDOM_ERASE: false, RANDOM_ERASE_VALUE: 0.5,\n",
            "    T_CROP: false, WORKERS: 1}\n",
            "\n",
            "05:28:44-147 \n",
            "05:28:44-149 \n",
            "05:28:44-152 ****************************************\n",
            "05:28:44-196 Found 1 GPUs\n",
            "05:28:44-203 No crawler necessary when using CIFAR10 dataset\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "05:28:45-373 Generated training data generator\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "05:28:46-251 Generated testing data generator\n",
            "05:28:46-254 Loaded vaegan_model_builder from models to build VAEGAN model\n",
            "05:28:46-316 Finished instantiating model\n",
            "05:28:48-738 None\n",
            "05:28:48-746 None\n",
            "05:28:48-752 None\n",
            "05:28:48-765 None\n",
            "05:28:48-770 Loaded VAEGANOptimizerBuilder from optimizer to build VAEGAN model\n",
            "05:28:48-772 Built optimizer\n",
            "05:28:48-776 Built scheduler for Encoder\n",
            "05:28:48-780 Built scheduler for Decoder\n",
            "05:28:48-781 Built scheduler for Discriminator\n",
            "05:28:48-782 Built scheduler for Autoencoder\n",
            "05:28:48-783 Built scheduler for LatentDiscriminator\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 64, 16, 16]           1,792\n",
            "            Conv2d-2            [-1, 128, 8, 8]          73,856\n",
            "       BatchNorm2d-3            [-1, 128, 8, 8]             256\n",
            "         ConvBlock-4            [-1, 128, 8, 8]               0\n",
            "            Conv2d-5            [-1, 256, 4, 4]         295,168\n",
            "       BatchNorm2d-6            [-1, 256, 4, 4]             512\n",
            "         ConvBlock-7            [-1, 256, 4, 4]               0\n",
            "            Conv2d-8            [-1, 512, 2, 2]       1,180,160\n",
            "       BatchNorm2d-9            [-1, 512, 2, 2]           1,024\n",
            "        ConvBlock-10            [-1, 512, 2, 2]               0\n",
            "AdaptiveAvgPool2d-11            [-1, 512, 1, 1]               0\n",
            "           Linear-12                  [-1, 256]         131,328\n",
            "================================================================\n",
            "Total params: 1,684,096\n",
            "Trainable params: 1,684,096\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 0.46\n",
            "Params size (MB): 6.42\n",
            "Estimated Total Size (MB): 6.90\n",
            "----------------------------------------------------------------\n",
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "   ConvTranspose2d-1            [-1, 512, 4, 4]       2,097,664\n",
            "       BatchNorm2d-2            [-1, 512, 4, 4]           1,024\n",
            "   ConvTranspose2d-3            [-1, 256, 8, 8]       1,179,904\n",
            "       BatchNorm2d-4            [-1, 256, 8, 8]             512\n",
            "       DeConvBlock-5            [-1, 256, 8, 8]               0\n",
            "   ConvTranspose2d-6          [-1, 128, 16, 16]         295,040\n",
            "       BatchNorm2d-7          [-1, 128, 16, 16]             256\n",
            "       DeConvBlock-8          [-1, 128, 16, 16]               0\n",
            "   ConvTranspose2d-9           [-1, 64, 32, 32]          73,792\n",
            "      BatchNorm2d-10           [-1, 64, 32, 32]             128\n",
            "      DeConvBlock-11           [-1, 64, 32, 32]               0\n",
            "  ConvTranspose2d-12            [-1, 3, 32, 32]           1,731\n",
            "================================================================\n",
            "Total params: 3,650,051\n",
            "Trainable params: 3,650,051\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 2.77\n",
            "Params size (MB): 13.92\n",
            "Estimated Total Size (MB): 16.70\n",
            "----------------------------------------------------------------\n",
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Linear-1                   [-1, 64]          16,448\n",
            "            Linear-2                  [-1, 128]           8,320\n",
            "            Linear-3                    [-1, 1]             129\n",
            "================================================================\n",
            "Total params: 24,897\n",
            "Trainable params: 24,897\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.00\n",
            "Params size (MB): 0.09\n",
            "Estimated Total Size (MB): 0.10\n",
            "----------------------------------------------------------------\n",
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 64, 16, 16]           1,792\n",
            "            Conv2d-2            [-1, 128, 8, 8]          73,856\n",
            "       BatchNorm2d-3            [-1, 128, 8, 8]             256\n",
            "         ConvBlock-4            [-1, 128, 8, 8]               0\n",
            "            Conv2d-5            [-1, 256, 4, 4]         295,168\n",
            "       BatchNorm2d-6            [-1, 256, 4, 4]             512\n",
            "         ConvBlock-7            [-1, 256, 4, 4]               0\n",
            "            Conv2d-8            [-1, 512, 2, 2]       1,180,160\n",
            "       BatchNorm2d-9            [-1, 512, 2, 2]           1,024\n",
            "        ConvBlock-10            [-1, 512, 2, 2]               0\n",
            "AdaptiveAvgPool2d-11            [-1, 512, 1, 1]               0\n",
            "           Linear-12                  [-1, 256]         131,328\n",
            "           Linear-13                    [-1, 1]             257\n",
            "================================================================\n",
            "Total params: 1,684,353\n",
            "Trainable params: 1,684,353\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 0.46\n",
            "Params size (MB): 6.43\n",
            "Estimated Total Size (MB): 6.90\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "05:28:49-263 Loaded VAEGANTrainer from trainer to build VAEGAN model\n",
            "05:28:49-272 Starting training\n",
            "05:28:49-273 Logging to:\tvaegan_cifar-v4-vaegan-cifar10-logger.log\n",
            "05:28:49-274 Models will be saved to local directory:\tvaegan_cifar-v4-vaegan-cifar10\n",
            "05:28:49-276 Models will be backed up to drive directory:\t./drive/My Drive/Vehicles/Models/vaegan_cifar-v4-vaegan-cifar10\n",
            "05:28:49-277 Models will be saved with base name:\tvaegan_cifar-v4_epoch[].pth\n",
            "05:28:49-278 Optimizers will be saved with base name:\tvaegan_cifar-v4_epoch[]_optimizer.pth\n",
            "05:28:49-279 Schedulers will be saved with base name:\tvaegan_cifar-v4_epoch[]_scheduler.pth\n",
            "05:28:49-351 Starting epoch 0 with 230 steps and learning rate 2.00000E-03\n",
            "05:28:59-623 Epoch0.24\tEncoder: 4.191 Decoder: 1.203 AE: 2.902 Discriminator: 1.246 Latent: 1.000\n",
            "05:29:10-236 Epoch0.49\tEncoder: 4.416 Decoder: 3.579 AE: 3.400 Discriminator: 0.172 Latent: 1.218\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GHtnlczvGnwB",
        "colab_type": "text"
      },
      "source": [
        "**bold text**# Encoder test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eH5ryr5s4h7K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!cp drive/My\\ Drive/Vehicles/Models/vaegan_cifar-v4-vaegan-cifar10/*epoch120.pth ."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5-zYfQOGogo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "config=\"config/vaegan_cifar10.yml\"\n",
        "#weights=\"vaegan_cifar-v3-vaegan-cifar10/vaegan_cifar-v3_epoch15.pth\"\n",
        "#weights=\"vaegan_cifar-v4_epoch120.pth\"\n",
        "cfg = kaptan.Kaptan(handler='yaml')\n",
        "config = cfg.import_config(config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54IySdSOGs2k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# --------------------- INSTANTIATE MODEL ------------------------\n",
        "model_builder = __import__(\"models\", fromlist=[\"*\"])\n",
        "model_builder = getattr(model_builder, config.get(\"EXECUTION.MODEL_BUILDER\"))\n",
        "\n",
        "vaegan_model = model_builder(   arch=config.get(\"MODEL.ARCH\"), base=config.get(\"MODEL.BASE\"), \\\n",
        "                                latent_dimensions = config.get(\"MODEL.LATENT_DIMENSIONS\"), \\\n",
        "                                **json.loads(config.get(\"MODEL.MODEL_KWARGS\")))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1b7PPERGzHj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5e31b6f3-da14-45e3-d892-6c40dfcb1b7b"
      },
      "source": [
        "vaegan_model.load_state_dict(torch.load(weights))\n",
        "vaegan_model.cuda()\n",
        "vaegan_model.eval()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VAEGAN(\n",
              "  (Encoder): Encoder(\n",
              "    (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "    (extractor): Sequential(\n",
              "      (0): ConvBlock(\n",
              "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): ConvBlock(\n",
              "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): ConvBlock(\n",
              "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (gap): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "    (embedding): Linear(in_features=512, out_features=256, bias=True)\n",
              "  )\n",
              "  (Decoder): Decoder(\n",
              "    (dconv1): ConvTranspose2d(256, 512, kernel_size=(4, 4), stride=(1, 1))\n",
              "    (dconv1_bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (upsampler): Sequential(\n",
              "      (0): DeConvBlock(\n",
              "        (dconv1): ConvTranspose2d(512, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))\n",
              "        (dconv1_bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): DeConvBlock(\n",
              "        (dconv1): ConvTranspose2d(256, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))\n",
              "        (dconv1_bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): DeConvBlock(\n",
              "        (dconv1): ConvTranspose2d(128, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))\n",
              "        (dconv1_bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (dconv4): ConvTranspose2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "  )\n",
              "  (LatentDiscriminator): LatentDiscriminator(\n",
              "    (dense1): Linear(in_features=256, out_features=64, bias=True)\n",
              "    (dense2): Linear(in_features=64, out_features=128, bias=True)\n",
              "    (dense3): Linear(in_features=128, out_features=1, bias=True)\n",
              "  )\n",
              "  (Discriminator): Discriminator(\n",
              "    (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "    (extractor): Sequential(\n",
              "      (0): ConvBlock(\n",
              "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): ConvBlock(\n",
              "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): ConvBlock(\n",
              "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
              "        (conv1_bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (gap): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "    (fc1): Linear(in_features=512, out_features=256, bias=True)\n",
              "    (fc2): Linear(in_features=256, out_features=1, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TpSEDdwuHIfo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "1d45e445-37ad-42aa-fb57-a6e8fa2677ac"
      },
      "source": [
        "NUM_GPUS = torch.cuda.device_count()\n",
        "NORMALIZATION_MEAN, NORMALIZATION_STD, RANDOM_ERASE_VALUE = utils.fix_generator_arguments(config)\n",
        "TRAINDATA_KWARGS = {\"rea_value\": config.get(\"TRANSFORMATION.RANDOM_ERASE_VALUE\")}\n",
        "\n",
        "# --------------------- BUILD GENERATORS ------------------------\n",
        "# Supported integrated data sources --> MNIST, CIFAR\n",
        "# For BDD or others need a crawler and stuff...but we;ll deal with it later\n",
        "from generators import ClassedGenerator\n",
        "\n",
        "load_dataset = config.get(\"EXECUTION.DATASET_PRELOAD\")\n",
        "if load_dataset in [\"MNIST\", \"CIFAR10\", \"CIFAR100\"]:\n",
        "    crawler = load_dataset\n",
        "    #dataset = torchvision.datasets.MNIST(root=\"./MNIST\", train=True,)\n",
        "else:\n",
        "    raise NotImplementedError()\n",
        "    # import crawler\n",
        "    # crawler = crawler...\n",
        "    # logger.info(\"Crawling data folder %s\"%config.get(\"DATASET.ROOT_DATA_FOLDER\"))\n",
        "    # crawler = ReidDataCrawler(data_folder = config.get(\"DATASET.ROOT_DATA_FOLDER\"), train_folder=config.get(\"DATASET.TRAIN_FOLDER\"), test_folder = config.get(\"DATASET.TEST_FOLDER\"), query_folder=config.get(\"DATASET.QUERY_FOLDER\"), **{\"logger\":logger})\n",
        "\n",
        "train_generator = ClassedGenerator( gpus=NUM_GPUS, i_shape=config.get(\"DATASET.SHAPE\"), \\\n",
        "                                    normalization_mean=NORMALIZATION_MEAN, normalization_std=NORMALIZATION_STD, normalization_scale=1./config.get(\"TRANSFORMATION.NORMALIZATION_SCALE\"), \\\n",
        "                                    h_flip = config.get(\"TRANSFORMATION.H_FLIP\"), t_crop=config.get(\"TRANSFORMATION.T_CROP\"), rea=config.get(\"TRANSFORMATION.RANDOM_ERASE\"), \n",
        "                                    **TRAINDATA_KWARGS)    \n",
        "train_generator.setup(  crawler, preload_classes = config.get(\"EXECUTION.DATASET_PRELOAD_CLASS\"), \\\n",
        "                        mode='train',batch_size=config.get(\"TRANSFORMATION.BATCH_SIZE\"), \\\n",
        "                        workers = config.get(\"TRANSFORMATION.WORKERS\"))\n",
        "\n",
        "test_generator = ClassedGenerator( gpus=NUM_GPUS, i_shape=config.get(\"DATASET.SHAPE\"), \\\n",
        "                                    normalization_mean=NORMALIZATION_MEAN, normalization_std=NORMALIZATION_STD, normalization_scale=1./config.get(\"TRANSFORMATION.NORMALIZATION_SCALE\"), \\\n",
        "                                    h_flip = config.get(\"TRANSFORMATION.H_FLIP\"), t_crop=config.get(\"TRANSFORMATION.T_CROP\"), rea=config.get(\"TRANSFORMATION.RANDOM_ERASE\"), \n",
        "                                    **TRAINDATA_KWARGS)    \n",
        "test_generator.setup(  crawler, preload_classes = config.get(\"EXECUTION.DATASET_TEST_PRELOAD_CLASS\"), \\\n",
        "                        mode='test',batch_size=config.get(\"TRANSFORMATION.BATCH_SIZE\"), \\\n",
        "                        workers = config.get(\"TRANSFORMATION.WORKERS\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./CIFAR10/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 99%|█████████▉| 169615360/170498071 [00:11<00:00, 17005822.24it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./CIFAR10/cifar-10-python.tar.gz to ./CIFAR10\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_dQ_kbGSHYq_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_batch = next(iter(train_generator.dataloader))\n",
        "batch = new_batch[0].numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e4fpnicS5zwQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_test_batch = next(iter(test_generator.dataloader))\n",
        "test_batch = new_test_batch[0].numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5tyPLwdoHx0D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03sEM-BeH-oL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "result = vaegan_model(torch.tensor(batch).cuda()).detach().cpu().numpy()\n",
        "encoding_disc = vaegan_model.LatentDiscriminator(vaegan_model.Encoder(torch.tensor(batch).cuda()).squeeze()).squeeze()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I0didlFn574w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_result = vaegan_model(torch.tensor(test_batch).cuda()).detach().cpu().numpy()\n",
        "test_encoding_disc = vaegan_model.LatentDiscriminator(vaegan_model.Encoder(torch.tensor(test_batch).cuda()).squeeze()).squeeze()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KdP8Us5ydfx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoded_value = vaegan_model.Encoder(torch.tensor(batch).cuda()).squeeze()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}