notebooks/ood_visualize_imagenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import matplotlib\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "\n",
    "import matplotlib as mpl\n",
    "\n",
    "from IPython.display import display\n",
    "\n",
    "import matplotlib.pylab as pylab\n",
    "\n",
    "from matplotlib import rc\n",
    "\n",
    "import torch\n",
    "from torch.nn.functional import log_softmax\n",
    "from sklearn.model_selection import KFold\n",
    "from scipy.optimize import minimize\n",
    "from matplotlib.lines import Line2D\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "\n",
    "ood_transforms = ['brightness', 'defocus_blur', 'fog',\n",
    "                  'glass_blur', 'jpeg_compression',\n",
    "                  'motion_blur', 'shot_noise', 'zoom_blur', 'contrast',\n",
    "                  'elastic_transform', 'frost', 'gaussian_noise', 'impulse_noise',\n",
    "                  'pixelate', 'snow',\n",
    "#                   'speckle_noise', 'spatter', 'gaussian_blur', 'saturate'\n",
    "                  ]\n",
    "names = {'ImageNet':{\n",
    "    'resnet50': {\n",
    "        'cropflips': ['ImageNet-resnet50-20-0_1_2_3_4_5_6_7_8-cropflips.npy',\n",
    "                      'ImageNet-resnet50-20-9_10_11_12_13_14_15_16_17_18-cropflips.npy'],\n",
    "        'optimal_policy':  ['ImageNet-resnet50-20-0_1_2_3_4_5_6_7_8-optimal_policy.npy',\n",
    "                            'ImageNet-resnet50-20-9_10_11_12_13_14_15_16_17_18-optimal_policy.npy'],\n",
    "        'central_crop': ['ImageNet-resnet50-1-0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18-centralcrop.npy'],\n",
    "    },\n",
    "    'efficientnet_b2': {\n",
    "        'cropflips': ['ImageNet-efficientnet_b2-20-0_1_2_3_4_5_6_7_8-cropflips.npy',\n",
    "                      'ImageNet-efficientnet_b2-20-9_10_11_12_13_14_15_16_17_18-cropflips.npy'],\n",
    "        'optimal_policy':  ['ImageNet-efficientnet_b2-20-0_1_2_3_4_5_6_7_8-optimal_policy.npy',\n",
    "                            'ImageNet-efficientnet_b2-20-9_10_11_12_13_14_15_16_17_18-optimal_policy.npy'],\n",
    "        'central_crop': ['ImageNet-efficientnet_b2-1-0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18-central_crop.npy'],\n",
    "    },\n",
    "    'tf_efficientnet_b5': {\n",
    "        'cropflips': ['ImageNet-tf_efficientnet_b5-20-0_1_2_3_4_5-cropflips.npy',\n",
    "                      'ImageNet-tf_efficientnet_b5-20-11_10_9-cropflips.npy',\n",
    "                      'ImageNet-tf_efficientnet_b5-20-12_13_14_15_16_17-cropflips.npy',\n",
    "                      'ImageNet-tf_efficientnet_b5-20-18-cropflips.npy',\n",
    "                      'ImageNet-tf_efficientnet_b5-20-6_7_8_9_10_11-cropflips.npy'],\n",
    "        'optimal_policy':  ['ImageNet-tf_efficientnet_b5-20-8_13_14_15-optimal_policy.npy',\n",
    "                            'ImageNet-tf_efficientnet_b5-20-9_10_11_12-optimal_policy.npy',\n",
    "                            'ImageNet-tf_efficientnet_b5-20-0_1_2-optimal_policy.npy',\n",
    "                            'ImageNet-tf_efficientnet_b5-20-3_4_5_6_7-optimal_policy.npy',\n",
    "                            'ImageNet-tf_efficientnet_b5-20-16_17_18-optimal_policy.npy'],\n",
    "        'central_crop': ['ImageNet-tf_efficientnet_b5-1-0_1_2_3_4_5_6_7_8_9_10_11_12_13_14_15_16_17_18-central_crop.npy'],\n",
    "    }\n",
    "},\n",
    "}\n",
    "\n",
    "short_legend_augs = {'cropflips': 'CF',\n",
    "                     'rand_optimalm': 'M*',\n",
    "                     'optimal_policy': 'GPS',\n",
    "                     'central_crop': 'CC'}\n",
    "model_names = {'resnet50': 'ResNet50',\n",
    "               'efficientnet_b2': 'EfficientNetB2',\n",
    "               'tf_efficientnet_b5': 'EfficientNetB5'\n",
    "             }\n",
    "colors_augs = {'cropflips': 0,\n",
    "             'optimal_policy': 2,\n",
    "             'central_crop': 3}\n",
    "colors_palette = [\n",
    "    '#bcbd22', '#e377c2', '#9467bd',\n",
    "    '#8c564b', '#1f77b4', '#d62728',\n",
    "    '#2ca02c', '#ff7f0e', '#7f7f7f']\n",
    "dataset_names = {'ImageNet': 'ImageNet',}\n",
    "\n",
    "matplotlib.rcParams['text.usetex'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = {'gaussian_noise': 0.886428,\n",
    "        'shot_noise': 0.894468,\n",
    "        'impulse_noise': 0.922640,\n",
    "        'defocus_blur': 0.819880,\n",
    "        'glass_blur': 0.826268,\n",
    "        'motion_blur': 0.785948,\n",
    "        'zoom_blur': 0.798360,\n",
    "        'snow': 0.866816,\n",
    "        'frost': 0.826572,\n",
    "        'fog': 0.819324,\n",
    "        'brightness': 0.564592,\n",
    "        'contrast': 0.853204,\n",
    "        'elastic_transform': 0.646056,\n",
    "        'pixelate': 0.717840,\n",
    "        'jpeg_compression': 0.606500\n",
    "       }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cropflips\n",
      "optimal_policy\n",
      "central_crop\n",
      "cropflips\n",
      "optimal_policy\n",
      "central_crop\n",
      "cropflips\n",
      "optimal_policy\n",
      "central_crop\n"
     ]
    }
   ],
   "source": [
    "metric = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))\n",
    "metric_severity = defaultdict(lambda: defaultdict(lambda:\n",
    "                                                  defaultdict(lambda: defaultdict(lambda:\n",
    "                                                                                  defaultdict(list)))))\n",
    "for dataset in names.keys():\n",
    "    for model in names[dataset].keys():\n",
    "        for aug in names[dataset][model].keys():\n",
    "            print(aug)\n",
    "            local_metrics = {}\n",
    "            for filename in names[dataset][model][aug]:\n",
    "                tmp = np.load('ood_res_imagenet/' + filename, allow_pickle=True).item()\n",
    "                local_metrics.update(tmp['ens_acc'])\n",
    "            for i in range(20):\n",
    "                aggr_er = 0.\n",
    "                for transform in ood_transforms:\n",
    "                    error = 0.\n",
    "                    for severity in range(5):\n",
    "                        num_ens = i\n",
    "                        if aug == 'central_crop':\n",
    "                            num_ens = 0\n",
    "                        loc_er = 1. - local_metrics[transform][str(severity)][str(num_ens)]\n",
    "                        error += loc_er / 5\n",
    "                    error /= base[transform]\n",
    "                    aggr_er += error / len(ood_transforms)\n",
    "                metric[dataset][model][aug].append(aggr_er)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAEeCAYAAAB45ojnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdfWwb150v/C9tJ07ciFSTtvDCHhXP4okVi3SaP8w4Ggd72xVryt5dbMzWonqBZ1eMTam7aKgAoQtcNGIaOsXilmwbGcVtRNqRFgUuRGej28WiJu2oTTe3GjXxRduNRCu+Qd80cuKkTSKOVMeOLc/zh3am4vuQ4otEfT8AEWuGc+Zwwt/Mbw7POWNSVVUFERERERHpNtW7AkREREREaw2TZCIiIiKiDFvqXQEqTFEUvPvuu7hx40a9q0JERESU5rbbbsOnPvUpmM3melel4pgkr2GKouCdd97Bjh07cOedd8JkMtW7SkREREQAAFVV8eGHH+Ly5csA0HCJMrtbrGHvvvsuduzYgW3btjFBJiIiojXFZDJh27Zt2LFjB9599916V6fimCSvYTdu3MCdd95Z72oQERER5XXnnXc2ZLdQJslrHFuQiYiIaC1r1FyFSTIRERERUQYmyUREREREGZgkExERERFlYJJMa0YgEICiKPWuRsPw+Xzw+Xz1rgZR1fG7vn4EAgHY7XbY7XYEAoGs9YlEAg6HI+96Kh/jpHRMkqnuZFlGKBRCLBard1WIqsbn86G1tTXrRbRR+Hw+zM/PY2RkBCdOnEA8HofH49HXS5KE/v5++P1+jIyMYHp6Om09Ua3xYSIb2MLCAv7lX/4FX/ziF9HU1FSXOkSjUYTDYQiCUJf9U2NSVbXgaOti66vF6XTi+PHjNd8vbTzvv/8+JEnC66+/jo8++gi333477r//foiiiLvvvrvm9VEUBZOTk7hw4QIAwGq1QhAEuFwuSJIEURQRjUZx7NgxdHZ2AgAGBwfhcDggyzKvEVQXbEnewF555RXMzs7ilVdeqVsdvF4vLl26hGAwWLc6UGN5+eWXkUgkoKpqzvWqqiKRSODll1+ucc2A5uZmCIKQ9iKqtDfffBPf+9738POf/xwfffQRAOCjjz7Cz3/+c3zve9/Dm2++WZd6dXV1pf1ttVoBAMlkEsByS/L+/fv19VqMSJJUu0oSrcAkeYNaXFzEL37xC6iqil/84hdYXFysd5WIVk1VVVy7dg2vvvpqzkRZS5BfffVVXLt2LW8iTbRevf/++zhz5gxu3LiBW7dupa27desWbty4gTNnzuD999+vab3MZnPOX1G0xxhribLNZktbLwiCvo6o1pgkbyC/+c1v8P3vfx/PPvssvvOd76St+853voNnn30W3//+9/Gb3/ymTjU0TpIkuFwutLa2wm63IxqNpq3PNUChtbUViUQi6312ux0OhwOhUAgOh6PsfqLRaBR2ux2tra1wOBx6nYzWNRqN6gNWotEoFEWBx+PRy1t5ofD5fPB4PGnbGBnkEgqF9EEzoVCorM+5lplMJnR2dmLfvn1ZifLKBHnfvn3o7OyseZeL6elpuFwu2O12+Hw+QwNV+V2v7Hc9X90bhSRJWFpaKviepaUlTE5O1qhG+cmyDEVRIIoiUqkUgD8lzZqmpibMz88XLGctxgjAOGkE7JO8gaRSKfzqV7/KWn7z5k0AwPz8PObn57Fnz55aV61ksizD7/dDEATIsgyPxwNBEPS+bEYEAgHMzc1hbGwMiUQC4XAYY2NjZf0ELkmSvr3ZbNZP/kbreu7cOezZswfDw8MYHR1FOBxGPB6H3++H3+/HwMAA+vv7MT4+nrZPURQxODiIs2fP4tSpUxAEAV6vN2cdfT4fLl68iMHBQf3zt7S0wO12l/x51zItUQaAV199FQDQ2dlZ9wQZWG4tCwaDEEURgUAAPT09GBsbK7gNv+uV+64LgpC37o3i9ddfz2pBznTr1i28/vrr+Ku/+qsa1Sq3UCgEURRhtVr1LhWKomQlygsLCwXLWWsxAjBOGgWT5A3kgQcewI0bN3Du3Dk9MV5py5Yt6OzsxAMPPFCH2pVmZWInCAKcTifOnj1b0kkxHo/jxIkT+kkkEolgenpa7ydXCu3ELgiC/t9S6mo2m/UTWV9fH06dOgWbzQZRFPUyMlsFRFHUt7FarZBlGbFYLOcJUZZlnDt3DmNjY/rn8/v9iEQiDZckA9mJspYs1zNB7u7uRnd3t/7/NBgMwuFw6Be2fPhdr9x3vbe3N2/dG4XWB7lS76uWaDSKixcvZt0kplKpnK3Jhay1GAEYJ42CSfIGY7fb8d577+G1115La23YtGkT7HY79u7dW8falSYajSIej0NRFMiyXDDRyMVisWT9nbnMKFEUYbFYYLfbIYpi2snKSF1X9sPTLhArT85G6rV//36cO3cu5zqtlcblchn/UOuclihrCTKAuiXIALL+n2sXnmQyWfS7y+96unK/68Xq3ghuv/12Qwnw7bffXoPa5CZJEiKRiN5SCfzp/3tmi+Xc3FxWP+Vc1lKMAIyTRsE+yRvQ1atXcw7ouHr1ap1qVDqtBe7EiRMYHx8vqzXU6XQiEolAlmUkEgmkUqmST6was9mM8fFxDA4OoqmpCeFwWO8DZ6SuuVpKVnOCzlfHS5cuZb0aldYHeaVCs16sVfyul1fHXN/1QnVvFPfffz82bSp8ad+0aRPuv//+GtUonSzL6O/vz+rKoCWAWt/kle8v1pq71mIEYJw0CrYkb0Bvv/02TCYTNm/erPeJWlpawltvvVXvqhmSTCYhyzKGh4f1k2yugR0r+7Hl6k918eJFWCwWOBwOCIKAkZGRrJ/5StXZ2YnOzk5IkgSPx2O4rpWQSCTyXkxEUYSiKEgmk2X/fLie5Bqkp/0N1L5FWVEUxOPxtIuh1pJT6ELM73puq/2uZ9a9kYiiiP/4j/8o2C958+bNaG9vr2GtlmkDz0ZGRnL+hG+1WpFIJPSYSCaT+sC+fNZyjACMk/WOLckb0LZt22C1WvGVr3wFf//3f4+vfOUraGtrw7Zt2+pWJ+2kldmKkIsW0NFoVL/rz/xJqbm5GbIs668nn3wyqxxZltHZ2Ynh4eGc8zTLsmx4RG8ikUAgEEAymUQymcTo6CgEQTBU13JJkoRYLAZZlhEIBCBJEnp7e3O+VxAEuN1u9Pf3Q5IkvS6NeOLLN4tFvlkvaiGVSiEQCCAUCiGZTEKSJAQCATidzoI3LfyuL6vUdz1f3RvJ3Xffja6uLtx2221ZLcqbNm3Cbbfdhq6urro8UKSnpwdOpxMA9P8HWtIILPeJjcViSCQSSCaTGBgYgCiKBf8frcUYARgnjYItyRtQZmLU3NyMI0eO1KUusVgsbfCBw+EAgLTBBLloAwzi8TgOHjyY1dLgdrsRj8fhcDggiiL8fn/WiaitrS1r4IMgCPodvjY62UhfLKvVirNnz6KnpweKosBqteojhovVtVyiKCKZTCIcDsNisSAYDBYcqBIMBhGNRhEIBPSfMPOdQNerQtO85Zv1ohYtyoIgYHx8HKFQCD09PbBYLIafvsfveuW+64Xq3kjuvfde/MM//AMmJyeznrjX3t5elwRZlmU96Tp16lTaumPHjuH48eMQRRHBYFDv9nDw4EFDD5paazECME4ahUldbx30NpCZmRns3r273tVoSNrPR+Pj4/odstZXDkDRabnqzefzYWFhAcPDw/Wuyprz8ssv49q1a3kTYC2RvuOOO/C5z32uDjWsLX7XiQpb7zECrI04acSchd0taEPS+mGt/AlJ+/lpo84H2Sg+97nPFWwh1lqUN0KCDPC7TlQMY4TyYZJMG1JnZyeSyaT+FCNguTUhGo3qfeZo/SrWhaJe08DVA7/rRIUxRigf9kmmDWllH9FIJKL3u/J6vQ35cA3auPhdJyqMMUL5sE/yGtaI/XuIiIio8TRizsLuFkREREREGZgkExERERFlYJK8xrE3DBEREa1ljZqrMElew7Zs2YKbN2/WuxpEREREed28eRNbtjTeXBBMktewO+64A4uLi/WuBhEREVFeCwsLuOOOO+pdjYpjkryGffKTn8Tvf/97XL16tWF/yiAiIqL1SVVVXL16FX/4wx/wyU9+st7VqThOAbfGpVIpvPfee7h+/Xq9q0JERESUZuvWrbjnnntgsVjqXZWKY5JMRERERJSB3S2IiIiIiDIwSSYiIiIiysAkmYiIiIgoA5NkIiIiIqIMTJKJiIiIiDIwSSYiIiIiysAkmYiIiIgoA5NkIiIiIqIMTJKJiIiIiDIwSSYiIiIiysAkmYiIiIgoA5NkIiIiIqIMTJKpqFgshtbW1rSXw+FAIBCAoigVLT+zPEVR0NraCkmSVr0fIxKJBBwOB+x2OwKBQE32SY0pV9ysfK38rieTSbhcLrS2tiKRSAAAQqEQWltb9e9ha2srYrFYXT5LPrWI3VLPP4FAAHa7nTG8QTHu6hN3jWpLvStA68eFCxdgNpsBLJ9choaG0NHRgR/96Ef68tUwm82IxWLwer2rLiuTz+fDuXPnspZfunRJ/7ckSejv78fg4CAEQcDAwAA8Hg+Gh4crXh/aOMbGxnLGx8pl/f398Pv96OzshKIokCQJZ86cwYULF/T3+P1+iKJoaJ8ulwsWi6Vm391qxq5Gi1VFUTA9PY1wOJx1/vH5fACAkZERyLKMgYEByLLMGN6AGHeVYTTuil1f1ysmyWTYypOL1WrFyZMn4XK58OSTT+LkyZOrLr+rqwuRSKRqAe90OnH8+PG866PRKI4dO4bOzk4AwODgIBwOB2RZhiAIVakTNT5BEAreRCaTSciyrH/vzGYzEokE2tvb07YrJS7cbnf5FS5DtWN3JbPZDFEUMTY2BrvdjqGhIRw/fhyKomByclJPcKxWKwRBgMvlgiRJhhMdagyMu8rKF3eaYtfX9YrdLWhV2tvbcfHixYqU1d3dDUVR9J+9yiFJElpbW3Oua25uhiAIaa/Mbffv36//rb2nVl09aGNKpVJZy+bn51dVptvtrugFu1BcAdWP3Xwyzz9dXV1p661WK4DlhIhoJcadsX3kkuu6X+z6ul4xSaZVmZycrFgwCIIAp9OJSCRSkfJKoV1EbTZbVp14gSUqrF6xOzk5iba2NgDLLV25WrIq0RWMaC1aC3HX6JgkU1lkWUYgEEAymUQwGExbFwqF9IEzoVBIXx6NRmG32/UBANFoNKvcvr4+JJPJoolprn14PB54PB4A0AcarLzDnp6ehsvlgt1uh8/nSxt8oLUqZF5Qm5qaVt26QJRPKBTK+s5qzp07lzWAKLPFKBqNwuFw6DGlDTDy+Xx6/9yV+8oVl9p7tbLsdntabBaLK001YzeTJElwuVyw2WwFf+KVZRmKorCrBaVh3FU+7gpdX9cz9kkmwzJ/knE6nRgfH09rSfb5fLh48SIGBwcBLI80b2lpgSAICIfD+mAK7eKVyWq1wmq1YmhoKG8/53z7GB4eRiKRQH9/f84BA1pCL4oiAoEAenp6MDY2lvYeRVGyEuWFhQUDR4coN7vdnrXM7XYjGAzi+PHj2LNnT87vrNPpLNjXPxQK4cyZMzhx4gSsViuSySSmpqZyvjdfzGg/DZ87dw579uzB8PAwRkdHEQ6H0dnZCUEQisaVppqxC2Sff7RjWEgoFIIoinq3C9o4GHfF61HJuDNyfV2PmCSTYVqCqygKXC4X9u/fn5Ygy7KMc+fOYWxsTL8o+f1+RCIR9Pb2wmw264MpCnXR6O3tRX9/f84kutA+3G533p9Wu7u70d3drbcoBYNBOByOrAE9qVQqZ2syUblyjbK3WCz6v8vpDqAoCk6dOoXh4WH9+ysIgj4IaaViMaPVQRv809fXh1OnTqUNWDVax2rErmZ8fFz/dzKZRCwWg8PhwODgYM4kOBqN4uLFiw1xoabSMe6M1aMScWf0+roeMUkmw1ZeiI4dO4ZwOJw2SEEb4OZyubK2FUURFosFdrsdoihCFMW8I3I7OzthNpsxNDSEvr6+tHWF9lFIZqBqJ6FkMqnXDUDWSWZubi6rnzJRKYqNsi+HFgdGLkBGYmbld1yrazk/l1YjdjUrb6y1xERrsVo5ZZe2r0gkkncaMGp8jDvj9SjESNwVu76uZ0ySqSzaXW80Gk1Lds1mc9YFSzM+Po5EIoGzZ88iHA5jamoq789Dvb29iEQi6O7uzlpXaB/l0m4AMkc8y7Jc82l9iCqtWMxU8teSWsau2+1GLBZDMpnUY1iWZfT392NsbKxhRtjT+rSR4q5RceAelcVsNsPv9yMcDut3vqIoQlGUggMIOjs7cfLkSQwPD+ecfFzjdruhKApGR0fTlhvZRyZFUbKemJSrRcBqtaYNWkgmkxz0Q2uS9p00Mj1hOTGzGpWM3WK0z68lw4qiwOPxYGRkhAkyVRzjbtnKuDN6fV2vmCRT2bxer/4TD7AcMG63G/39/ZAkCbIsI5FIwOPxIJFI6LNhJJNJjI6OFryImc1muN1unDlzJm15oX1o64Hlx0tLkgRJkpBKpRAIBBAKhZBMJiFJEgKBAJxOZ9pdsN/vRywWQyKRQDKZxMDAAERR5MWWVmV6elr/3q98rWb0t9lsxrFjx9Df369/XxOJRM6fVIvFjBG54qpQ3SoVu/nIsoxQKIRwOAy/36//VN3T0wOn0wkAacdalmXDn5UaA+POWD1WG3dGr6/rFbtb0Kr4/X4EAgF0d3dDEAQEg0FEo1EEAgHIsgyr1Yre3l5YrVacPXsWPT09UBQFVqtVH2mbj9frzbpDBZB3H8BywFutVvT398NqtcLv90MQBIyPjyMUCqGnpwcWiyXn04FEUUQwGEQkEoEsyzh48GDR0fNExeS7KA4ODuYc8GPU8ePH0dzcjHA4rMdBvq5BhWLGiFxxVUilYnellaPstcG/KwdQybKsJ0KnTp1K2/bYsWMN+TQwyo9xZ6weq407o9fX9cqkqqpa70oQEREREa0l7G5BRERERJSBSTIRERERUQYmyUREREREGZgkExERERFlYJJMiMViWc9nX2u0Ora2tmZN4aMoClpbWw3NXbla0WgUDocDra2tsNvtcLlcaXMrE5WCsWccY48qibFn3EaOPSbJtK6YzeacU9xUm6IocLlc+vPux8bGMDg4CJvNhqmpqZrXh6jWGHtE9cHYqx/Ok0zrSldXFyKRSNqjsGvhySefhCzLWY+6bYQnChEZwdgjqg/GXv2wJZnWle7ubiiKsqqfeiRJKulnNlmWce7cOZw4cYJP36MNi7FHVB+MvfphkkzriiAIcDqdiEQiNdvn6OgozGbzqp7SRLTeMfaI6oOxVz9Mkiknn88Hn8+nd9i32+2IRqNQFAUejwetra1wOBxIJpP6NpIkweVy6Z37o9FoznLtdjscDgdCoZA+GEATCoVgt9tht9sRCoVy1q2vr09//GwhucryeDz640q1ARHF7s4vXrwIm81W8D1ElcLY+xPGHtUSY+9PGHv/SaUNb3R0VN21a1fasscee0zdtWuXGolE1NnZWfWb3/ymumvXLvXw4cPqxMSEOj09rR4+fFjt6OhIK2diYkKdnZ1VJyYm1F27dqnxeFxfPzAwoB4+fFidnZ1VI5GIumvXLnV6elpNpVL6Pjs6OtSJiQl1YmJC7ejoUEdHR3PW8fDhw+pjjz2mqqqqplIpddeuXerExERa/fOVFY/Hsz5vIR0dHfq+iCqJsVcYY4+qhbFXGGNvGZNkynuy2Lt3r/63FpADAwMFt8ssY2WQ7d27N+3ksXfvXj2AZ2dn9ZOHJh6Pq4cPH865Ly3gU6lU1smiWFnaicyonp4etaenx/D7iYxi7BXG2KNqYewVxthbxtktKK+VP7WYzWYAgNVq1ZdZLJasbaLRKOLxOBRFgSzLaaNgM99vsVj0Zdpcjy6Xy1DdOjs7YTabMTQ0hL6+vrR1pZZVTFtbG86cOVORsoiMYOwtY+xRrTH2ljH2ljFJpryampqyluU6QWgcDgcEQcCJEydgtVoRCAQgy7K+Xht4YLVakUwmkUql0k4mZrMZFy5cMFy/3t5eRCIRdHd3Z60rtSyXy5VW14MHDyIYDAIADh06hFOnTiGRSGz4QQxUG4w9xh7VB2OPsbcSk2SqiGQyCVmWMTw8rE8XMz8/n/aeixcvwmKx6CeVkZER/U5dFEUoioJkMpl2116I2+1GOBzG6Oho2vJyyhoZGcm7zmq1wul0YmBgAFardUNPh0NrD2OPqD4Ye42Ps1tQRWhBGY1GIcsyEokEzp07l/YeWZbR2dmJ4eFh/W5VIwgC3G43+vv7IUmSXoY2IjcXs9kMt9ud9ZNQsbK0YE8kEpAkCZIkwWw2Z71WeuaZZ2Cz2eByuRCNRpFMJiFJEkKhUMV+3iIqB2OPqD4Ye42PLclUMX6/H5FIBPF4HAcPHsx6Kk9bWxsCgUDaMkEQ9LvwYDCIaDSq/1xltVrR29tbcJ9erzfn4zoLlSUIAqxWK/r7+2G1WuH3+4t+NrPZjOHhYb3vWTgchtlshs1mw4kTJ4puT1RNjD2i+mDsNTaTqqpqvStBjU+SJHg8HoyPj+t3tLIso7+/HwAwNjZWz+oRNSzGHlF9MPbWP3a3oJrQ+kmt7Nek/TykKEoda0bU2Bh7RPXB2Fv/mCRTTXR2diKZTOpPLwKW77Kj0SicTmeda0fUuBh7RPXB2Fv/2N2CakaWZYRCIUxOTkJRFFitVrjdbrjd7npXjaihMfaI6oOxt74xSSYiIiIiysDZLargl7/8JbZu3Zp3/fXr1wuup3Q8XqUp9Xhdv34dDzzwQBVrVDuMvcri8SrNRo29YnEH8LtUKh4v48o5VkZjj0lyFWzduhW7d+/Ou35mZqbgekrH41WaUo/XzMxMFWtTW4y9yuLxKs1Gjb1icQfwu1QqHi/jyjlWRmOPA/eIiIiIiDIwSSYiIiIiysAkmYiIiIgoQ8MkyYlEAg6HA3a7PesRkOUKBAKw2+0VLZOKKzbhCidkWVvKjb1oNAqXywW73Q6Px5O2LhaL6WX6fD5OvF8jjL31pRqxpwkEAoy7GmLsrU0NkSRLkoT+/n74/X6MjIxgeno6b+Br729tbc350rbz+XyYn5/HyMgITpw4gXg8XrBMqoxf/3oQb775TN4TgqqqePPNZ/DrXw/WuGaUS6mxpwkEAohEIujt7cXY2Bg6Ozv1dbFYDOFwGMFgUH9sq8vlqtpnoGWMvfWlGrEH/Gle31gsVq2qUwbG3trVELNbRKNRHDt2TA/2wcFBOBwOyLKc9jhIjSiKOZ+ZPjAwAFEUoSgKJicnceHCBQDQHyvpcrkgSRJEUazuB9qgVFXFzZsK5LkRAMC99z4Jk8mUtv7NN5+BPDcCYWcPVFVNW0+1V2rsAcsX91gshrGxMVitVgBIm1g/Go3C7/frcXby5Em0trYikUhkXdCpMhh760+1Yi8cDufdniqPsbe2NUxL8v79+/W/BUGAIAiQJCnvNlarNe0lyzIAwOv1AgC6urqy3g8sP4udqsNkMuHee5+EsLMH8txI2p115oki80RC9VFO7EWjUYiiqMdUJlmWYbPZ0pYJgoBUKlWZSlMWxt76U43Y83q9uHTpEoLBYMXrS7kx9ta2dd+SrCWtuS6qRhNaRVEwMDCAkZERAIDZbMbx48ez3mc2m1dXWSpKO2EA0O+sVfULPFGsQeXGniRJBS/CZrMZsiynXchTqRR/wakyxt76Ua3Yo/pg7K1d6z5J1lqXMhPYpqYmzM/PGypjaGgINpst7901sNy6pSgKL9Q1kH3CGAEAnijWmHJiT/vFxmKxIBAIIB6Pw2KxIBgM6rHl9/sRDof1bk7RaBQnTpzgT8A1wNhbH6oVe1Q/jL21ad0nyRpFUbJOGAsLC4a2O3XqFIaHhwu+LxQKFfyZaqXr168XfJrLtWvXGuZJS9Wkql+AdqIAgJs3v4A33nijbvVZL2r9/Sol9rQLdTgcht/vh9frRTQahcfjwYULF2A2m+F2u6EoChwOB4DlRGBw0NiAFcZeZTD2yrPeY69cxeIOYOwZxdgrXTW/Ww2TJKdSqZx31cXEYjGYzeaCd9LRaBQXL17MOdgvFz4ad/W0vlgrbdnyIu+oDaj1o3FLiT2LxQJgue+jNuAoGAwiHo8jFovB6/UiFoshFothcHAQgiBgaGgIHo8nbbBRPoy91WPslW+9x165+FjqymDslYePpS5AC/zM+Rzn5ubQ3NxcdHtJktDe3l5wfSQSwfDwMPsk10jmYIU/2/5vOQc1UH2VE3tal4nMvpQ2mw1TU1NQFAWBQAB+vx+dnZ2wWq04efIkzGYzp6SqAcbe+lCN2KP6YuytTes+SdZaljJHvmcO/Mknc4RwZhn9/f0YGxtjf8gayTeaN9/oX6qfcmJPu9HMNVNFc3Mzpqen08rW2Gw2w2MMqDyMvfWjGrFH9cPYW7vWfZIMLJ8wEomE/ncymTQ0yC7fCGFg+Q7d4/FgZGSECXKNFJruhieMtamc2HM6nZiYmEhbNj09DavVqsei1n9y5fp8N7O0eoy99afSsUf1wdhb2xqiT7Lf74fH44EoihAEQX8oiJbcyrKMRCKR1edKu6POlQT39PTA6XQCSJ8b2Ww2M2muEpPJhC1bzHlH864c/btli5l9tNaAcmKvr68PLpcLLS0tsNlsGBoagsVi0R9q4Ha7EQgEEAwGIQgCQqFQ2nqqPMbe+lON2NNo3Thy9XmmymLsrXFqgxgdHVUPHz6s7t27Vx0YGMhat2vXrpzb5Fo+Ozur7tq1K+frm9/8ZtG6XLx4cVXrN7pbt26l/Z15vDLXU7pSv1+r/T6WE3sTExNqR0eHumvXLvWxxx5TU6lU2vpIJKJ2dHSoe/fuzbk+H8be6jD2Vme9x562TeZrenp61Z+DsVcYY6985Xy3jG5jUlW23VdasZGWHOVbGh6v0pQzwr5Rji9jr7J4vEqzUWPPyOdolM9aKzxexpU7u4WRbRqiTzIRERERUSUxSSYiIiIiysAkmYiIiIgoA5NkIiIiIqIMTJKJiIiIiDKsiXmSFxcX9adpNTc346677qpzjYiIiIhoI6tLkjwzM4PR0VFMTk5mPVlLIwgC2lWmP1EAACAASURBVNvb8aUvfQn33XdfjWtIRERERBtZTZPkmZkZ+Hw+zM3Nob29HV1dXbBarbBYLGhqagIALCwsIJVKIZlMYmpqCo888ghaWlpw8uRJJstEREREVBM1S5JPnTqFSCQCv9+Prq6uou9vb2/X/x2LxfB3f/d3+PKXv4xHH320mtUkIiIiIqrdwD1JkvCjH/3IUIKcye12Y3x8HD/96U+rUDMiIiIionQ1a0l+/vnnV7W92WxedRlEREREREbUdQq406dPY3FxsZ5VICIiIiLKUtckeWJiAvF4vJ5VICIiIiLKUtck2ev1IhwO4/Lly/WsBhERERFRmro+TCQej8NsNsPhcKCtrQ2CIKStf/bZZ+tUMyIiIiLayOqaJKdSKbS1taGtrU1fpqpqHWtERERERFTnJHlwcBALCwsIhUKYnJyEyWRCe3s7jh8/zkdTExEREVHd1LVP8sLCAhwOB6anp+F0OnHgwAFMTU2ho6OD/ZSJiIiIqG7q2pI8MDAAp9OJYDCYttzn8yEQCOD06dN1qhkRERERbWR1bUmWJAm9vb1Zy/v6+jA9PV2HGhERERER1TlJtlgsmJuby1quKAosFksdakREREREVOckuaurC4FAIK3/8czMDJ566ik4nc461oyIiIiINrK69kn2er2QZRkdHR16y7GiKHA6nXjiiSfqWTUiIiIi2sDq2pIMAMFgEC+99BKeeOIJPPHEE3jxxRfLeohIIpGAw+GA3W5HIBCoaB0DgQAURalomUSNotzYi0ajcLlcsNvt8Hg8VawhUWOqxnUvkUjocVnpaynRelPXJPn06dNYXFyEIAjo6upCV1dX2oNFjJIkCf39/fD7/RgZGcH09HTBi64kSWhtbc35WrmdLMsIhUKIxWJlfT6iRldq7GkCgQAikQh6e3sxNjaGzs5OvTwjsUm00ZUTez6fL2dsaRKJBPr7+9Hb26uXyUSZNrK6dreYmJiA2WzGkSNHVlVONBrFsWPH9Avt4OAgHA4HZFnOetQ1AIiiiLGxsazlAwMDEEVRLzMcDufcnoiWlRp7wPLFPRaLYWxsDFarFQDgdrsBGItNIiov9gDA6XTi+PHjOdeFw2H4/X69zBMnTsDlcsHv98NsNlf+QxCtcXVtSfZ6vQiHw6t+cIgkSdi/f7/+tyAIEAQBkiTl3cZqtaa9ZFnW66T999KlS1lzOBPRn5QTe9FoFKIo6glypmKxSUTlxR4ANDc36+/VXsDyL6eyLKfdjGoxGo/Hq/AJiNa+urYkx+NxmM1mOBwOtLW1Zd39GumbnEwmAQA2my1tuSAI+rpiFEXBwMAARkZGjFWciMqOPUmSDN98MjaJslXiupdJuxnNvA4LgsAxObRh1bUlOZVKoa2tDQcOHMDOnTuhqmray2gZALJ+CmpqasL8/LyhMoaGhmCz2fK2bBFRtnJiT7sQWywWBAIB2O12OByOvK1fjE2ibKu57k1PT+sD83w+n54AazNMaWWvpMUt0UZT15bkwcHBipWlKErWCWNhYcHQdqdOncLw8HDF6nL9+nXMzMzkXX/t2rWC6ykdj1dpan28Sok97WKr9X30er2IRqPweDy4cOFCWjnlxCZjr7J4vEqzlmNPk0wmEQwGIYoiAoEAenp6MDY2prcgS5KkjxFQFAWyLBdNvIvFHcDvUql4vIyr5rGqa5J8+vRpuN1u3HXXXasuK5VK5byrLiYWi8FsNld0UNDWrVuxe/fuvOtnZmYKrqd0PF6lKfV4rfbkUkrsaa1VXq9XHxwUDAYRj8cRi8XS+h2XE5uMvcri8SrNWo49AOju7kZ3d7ceU8FgUP8lRxRFOJ1OhMNhiKIIi8WCJ598EkB2F4xMxeIO4HepVDxexpVzrIzGXl27W0xMTKx6QMDKh5CsNDc3h+bm5qLbS5KE9vb2VdWBaCMqJ/a0i21mX0qbzYapqam0ZYxNotzKve6Joph206nFo9aP+eTJk7BYLHA4HHC5XDh06BDMZjNaWloq/RGI1oV1P7uF1lcxsx+VLMuG+jFmjhAmImPKiT2t1StXv8fMiztjkyi31V73ChkfH8eFCxcwPj4OURShKAqnX6QNa93PbgEsnzASiYQeyMlk0lBg5xshTETGlBN7TqcTExMTae+Znp7Wu19o5QCMTaJ8So09RVEQj8f1/sYA9AGzmdtoN7NDQ0MQRZHPC6ANq65Jsja7hfaUPaMzWmTy+/3weDx6MGsPHlg5/2MikciaZ1W7Cy90AtB+zsrV94tooysn9vr6+uByudDS0gKbzYahoSFYLJa0i7eR2CTayEqNvVQqhUAggNnZWRw6dEj/2+l06q3PiUQCwHICPjo6ijNnzuR8uA9Vx9Wrv8OsfBpXrvwrlpb+iHfe/Ri2b/9btAhHsW3bp+tdvQ2pIWa3EEURwWAQkUgEsizj4MGDafOwSpKEcDiclSRrI+1zJb+xWCztcZwOhwMA0p4SRrTRlRN7VqsVw8PDCAQCkGUZTqcz60JcKDaJqPTYEwQB4+PjCIVC6OnpgcViyXr6ntlsRjgcRjKZ1J9+yRvV2vjDez/B1NRXoKo3oKo3AQBLS4t4660Y3n57DHv2fBefuOez9a3kBmRSy22+LdHi4qLhWSwWFhYwOTmJAwcOVLlW1VFspCVHrZaGx6s05Yywb5Tjy9irLB6v0mzU2DPyORrls1bD1au/w6uv/RVu3fow73s2bboT+x78IVuUcyh3dgsj29Rs4J7dbsfc3FzW8qeeegqLi4tpy2RZRn9/f62qRkRERFQXs/JpqOqNgu9R1RuYlZ+vUY1IU7MkOV+D9ZkzZww/GY+IiIiokVy58q96F4t8VPUmrlz5QY1qRJq6TgEHlD9Yj4iIiGi9W1r6Y0XfR5VT9ySZiIiIaKPavPljFX0fVQ6TZCIiIqI62b79b2EyFZ5szGTagu3bH6lRjUhT0yngFhcX0wbpaV0tMpfnehoXERERUaNpEY7i7bfHCvZLNpluQ4vwaA1rRUCNk+TDhw8bWq6qKkwmUy2qRERERFQ327Z9Gnv2fDdrnmRguQXZZLoNe/Z8l9O/1UHNkuSVk5wTERER0bJP3PNZ7Hvwh5iVn8eVKz/A0tIfsXnzx7B9+yNoER5lglwnNUuSu7q6arUrIiIionVl27ZP477Wp3Ff69N8+MoaUTRJPn36NARB0F9Gn5pHRERERLReFZ3dIhQKQZZlzM/PY2pqKm3d5ORkzqfoERERERGtZ0VbkkVRxNGjR3Oua29vx7lz5xCLxbBnzx4cOHCg4hUkIiIiIqq1oi3JO3fuLLje6XTiiSeewA9/+EM4nc6KVYyIiIiIqF6KtiSbzWb93xcvXsTZs2fx8Y9/HKIopnUqHxwcxIMPPlidWhIRERER1VDRJHnlfMVtbW1oa2vDgw8+CLPZDFmW07pYtLe3V6eWtOFcvfo7zMqnceXKv2Jp6Y94592PYfv2v0WLcJRT4RAREVHVFU2SFUXJWtbZ2YkjR45kLRcEoeQKzM3NQZZlLCwsZK1jH+eN6Q/v/SRrUvWlpUW89VYMb789hj17votP3PPZ+laSiIiIGlrRJDkej8NisUAURTz00EMAgE9/OndLXqlPyTt16hS+9a1v6Y+nzixrZmampPJo/bt69XeYmvoKbt36MGudqt6Eqt7E1NRXsO/BH7JFmYiIiKrG0MNE4vE4IpEITCYTRFEEAOzfvx/33XffqnYeiURw9OhR9PX1oampaVVlUWOYlU9DVW8UfI+q3sCs/Dzua326RrUiIiKijaZoktzZ2YlgMAhFURCPxyFJEiYnJ/HII4/oSfP+/fvR3t6es2tGIRaLBd3d3UyQSXflyr+mPbc+F1W9iStXfsAkmYiIiKqm6BRwx48fB7A8y4Xb7cbg4CBee+01vPTSS/j617+Ou+66C8899xwOHz6MM2fOlLRzt9uNc+fOlVdzakhLS3+s6PuIiIiIylG0JTlfK68gCHC73XC73QAAWZbR399f0s6npqZw/vx5RCIR2Gy2rH09++yzJZVH69/mzR/D0tKioffRMlVVC44HKLaeiIiIshnqk2yEIAh6f+VSrJzBItcAPtpYtm//W7z1VqxglwuTaQu2b3+khrVau177t1/j+oc38fCRe3Mmwqqq4qcvvImtd27Bg3/z53WoIRER0fpUsSQZAPx+f0nvHxwcrOTuqQG0CEfx9ttjRZLk29AiPFrDWq1Nqqri+oc38fqP5wAgK1HWEuTXfzyH+/9yJ1uUiYiISlC0T/Lk5GRVK7C4uIinnnoKBw4cwIEDB/D1r38di4vFf27PlEgk4HA4YLfbEQgEKlK3apRJhW3b9mns2fNdbNp0J0ym9Hs4k2kLNm26E3v2fJfTv2F5msSHj9yL+/9yJ17/8Rx++sKb+q8xmQlyvpbmSig3TqLRKFwuF+x2OzweT8nriTY6XqOIqstQS/Lu3bvR1tYGURThdruxc+fOiux8YWEBHR0d2Llzp97tQpIkdHR04MUXXzS8H0mS0N/fj8HBQQiCgIGBAXg8HgwPD+d9f76LriiKGB4eLrlMqpxP3PNZ7Hvwh5iVn8eVKz/A0tIfsXnzx7B9+yNoER5lgryCligD0FuU77HVLkEuN04CgQDi8ThOnDgBq9UKSZJKWk+00ZUTez6fL+dg+UuXLhm6LhJtNEWT5Pb2dhw5cgQ2mw02my2t37Asy/j4xz+Ou+66q6ydDwwM6FPMreTz+fDUU0/h9OnThsqJRqM4duwYOjs7ASx343A4HJBlOedTAEVRxNjYWM76aP2qSy2TKmvbtk/jvtancV/r05iZmcHu3bvrXaU1KytR/vHy8monyEB5cSJJEmKxGMbGxmC1WgFAHwBsZD0RlX+Ncjqd+qxVKxm5LhJtNEW7W7zwwgv46le/iq6uLrS1taUFnyAIOHv2LM6fP1/WziVJQm9vb9byvr4+TE9Pl1TO/v370+olCELB1ier1Zr2kmUZAOD1essuk6heVibKmmonyEB5cRKNRiGKop4Al7qeiMq/RjU3N+vv1V6aYtdFoo2maJKsKErBluKuri7s3r0bL7zwQsk7t1gsmJuby7lPi8ViqIxkMgkAsNlsacsFQdDXFaMoCgYGBnDixImKlUlUS1of5JVW9lGuhnLjRJIkvfWrnPVEG10trlGZ10Wijahokjw/P1+0EEEQsHPnzpwJbyFdXV0IBAK4fPmyvmxmZgZPPfUUnE6noTJSqRSA5YedrNTU1GSo7gAwNDQEm82mt1xVokyiWskcpPc53/acg/kqrZw40VqmLBYLAoEA7HY7HA6H3vpVbD1V39Wrv8MblwL4yb9/Bm+9/Tf4yb9/Bm9cCuDq1d/Vu2r0n1ZzjZqentYHxPp8vrxPys28LhJtREX7JBt91HR7eztOnz6No0ePGt651+uFLMvo6OjQW44VRYHT6cQTTzxhuBxtu8wTxsLCgqHtTp06lXNQQrllXr9+HTMzM3nXX7t2reB6SsfjlZ+qqnjzFQVzv7yKnQ9swz22m7h+/TrusQE739+G1388h/fffx/3/oW5al0vSokTLQkOh8Pw+/3wer2IRqPweDy4cOFC0fWZ+8nE2Fuda9f+Dz744J+g4iaAJQDA0tIiLl8exVuX/wUf//h/wx137K1vJdewWn+/yrlGJZNJBINBiKKIQCCAnp6erL7Iha6LuRSLO4CxVyoeL+OqeawMPXFvbm7O0EwT5bRYBYNBeL1eSJIEk8kEm82Gtra2kstJpVI576qLicViMJvNOQcmlFvm1q1bCw4040C00vB45aa1IM/98mraID3teO3e/acW5rvvvjtvH+XVnlxKiRPtZtjr9epdKoLBIOLxOGKxmB6H+dYX6xvJ2Cvf1au/w6uv/XeouJ5j7RJULGE+9d+xr/WHnGEmj1K/X7WMPQDo7u5Gd3e3HmfBYFD/pWblNbDQdTGXYnEHMPZKxeNlXDnHymjsFe1u4Xa7DT9uWvsJqFTaI661wYGlWNkCvdLc3Byam5uLbi9JEtrb2ytaJlEtmEwmbL1zS95ZLFbOo7z1zi0Vb0kuJ060QUKZfSltNhumpqaKrqfqmZVPQ1VvFHyPqt7ArPx8jWpE+ZR7jRJFMS3x1eItsx9zrusi0UZUtCVZEAQ89NBDePzxx/Hss88WfG+xrhlHjx5FZ2cnjhw5AgB4/PHHC76/2P4AZPUj1siybGjaKEmSsqagW22ZRLXy4N/8ecEn6WmJcjW6WpQTJ1qrV64b6ubm5qLrqXquXPnXgk+6BABVvYkrV36A+1qfrlGtKJdqX6NyXReJNqKiLckAcPz4caRSKRw4cAA/+9nPcr7nhRdeKNoV4YMPPkj7W1XVgi+jrFYrEomE/ncymYSiKEV/Kso3Qng1ZRLVWrEEuJrTwJUTJ06nExMTE2nLpqen9Qt/sfVUHUtLf6zo+6i6So09RVEQi8XSlmkDYlduU+i6SLTRGHriHgAMDw/D5/Ohp6cHLS0taG9vR0tLC4DlQJNlueh8yZmDAwYHB8uocja/3w+PxwNRFPUnD2n/BpbvrhOJRFZ/Ru0uPNfE68XKJKLyYq+vrw8ulwstLS2w2WwYGhqCxWLRW8CKrafq2Lz5Y1haWjT0Pqq/UmMvlUohEAhgdnYWhw4d0v92Op1pN6CFrotEG42hlmTNyZMn8fzzz2PHjh2IxWIIhUIYGhrCXXfdhRdffLHknedLqicnJzE5OWm4HFEUEQwGEYlE0NPTA5vNljYqV5IkhMPhrO20kfS5RswXK5OIyos9q9WK4eFhRKNRuFwuAOk30MXWU3Vs3/63MJkKt5uYTFuwffsjNaoRFVJq7AmCgPHxcciyjJ6eHj1BPnnyZFq5ha6LRBuNSV3FJKoLCwuGZnvIZ/fu3TlHGE5OTiIcDpeVeK8FxUZactRqaXi8SlPOCPtGOb6FPsviB+/j+//tcfx///Qs7vr43TWu2dq3PLvFX+HWrQ/zvmfTpjux70HObpHPRo09I5+jUT5rrfB4GVfu7BZGtimpJTmTliBPTk6W9WjqfPm5oii4ePHiaqpGRJTmZ2MxXJ3/AK/+rzP1rsqatG3bp7Fnz3exadOdWS3KJtMWbNp0J/bs+S4TZCLaMAz3SdYsLi7qcyjabDbs3r0b7e3tkGUZL7zwgj5zRSH33XcfTCYTTCZTzkxeVVUO0iGiivnj/AdI/uQlQFUx/fJ5tH+hG9ssnC0j0yfu+Sz2PfhDzMrP48qVH2Bp6Y/YvPlj2L79EbQIjzJBJqINpeQk+Wtf+xosFgvm5uYQCoVgMpnQ1dUFURQNP51veHgYqqri0UcfxeDgYFbfp507d3LQABGtyhsTryD57+P44O3LWHj/PX2WD1VVMfSPPWi6+x58/M92wPpfHLhv/1/UubZrx7Ztn8Z9rU/jvtan+ZMvEW1oJSfJ+/fvR1dXl/63JElIJBIIhUL40pe+ZKgMbZJyURThdDpLrQIRUVEffXgVv339F0BGt66lG8sPzEi9+w5Sv38X9+7bX4/qERHRGldykqw96UeT+QSfUjz/PJ/cRETVcb+jE7du3cK/f/8Ubn70Udb6LbdvxWf/7hju7+CNOhERZSt54F5bW1tJ07MVc/r0aRw4cAC7d+/G7t278YUvfAEvvfRSxconoo3rgQOH8JnPH8KmzZvTlm/avBmfOXAQn/n8wTrVjIiI1rqyZrcIhUI4evQozp8/j8XF4pPP5xMIBBAOh7F79248/fTTePrpp2E2m+Hz+fDCCy+UXS4RkeaqksKtpaW0ZbeWlvChwTEURES0MZXc3SIcDuPQoUOYn5/Hc889B5/Ppz+Br7OzU+9vbEQikcCxY8fwxBNP6Mu6uroQCAQQjUYNzZRBRFTIu7/5FUybNmHT5i1o3tmC+blZ3Fq6iXd+86t6V42IiNawVQ/cA5bnSZ6YmEAsFispSbZYLDh06FDWcq/Xm/ZMeiKici3dvIFdDz2Mv/ivPbj8h/ew4xP34JX/OYJ3fv1mvatGRERrWMlJci7t7e0lJccat9sNSZKyphj62c9+hoMH2VeQiFbv6GBU//flP7wH8yc/hb/u/2oda0REROtByUlye3s7zp8/jwMHDqx655IkYXJyEpIkZS0XRRFHjx5NW3769OlV75OIiIiIqJiSk+TJyUmEw2EMDQ3h0KFDEEWx7Mnmm5qacibblUjAiYiIiIjKVXKSrCgKRkZGMD09jYmJCQwNDWFhYQGiKMLtdpeU4A4ODpa6eyIiIiKiqis5STabzUilUujq6tIH8MmyjMnJSUxNTbEVmIiIiIjWvZKT5K6uLj0p1gbrCYIAQRBK3vn58+cLrmfCTURERET1UNbsFuUmxZl8Pl/O5SaTCQAwMzOz6n0QEREREZWqIlPAleuNN97IWqYoCgYGBvDlL3+5DjUiIiIiIirzsdTVZDab4ff7EQ6H610VIiIiWqXFD97HTwb/CYsfvF/vqhCVZM0lyZrMuZMbBU8WRES0kfxsLIar8x/g1f91pt5VISpJXbtbPP7441nLFhYWIMsyzGZzHWpUfStPFh2PsksJERE1rj/Of4DkT14CVBXTL59H+xe6sc3SXO9qERlS1yRZVdWsZXfddRceeughdHd316FG1cWTBRERNbo3Jl5B8t/H8cHbl7Hw/nv6YHxVVTH0jz1ouvsefPzPdsD6Xxy4b/9f1Lm2RPnVNUneCA8T4cmCiIg2ko8+vIrfvv4LIKMhbOnGDQBA6t13kPr9u7h33/56VI/IsLr2ST59+jQWFxfrWYWq004WqXffwa2bN/WTxNKNG7h18yZS776D377+C3x07cM615SIiGj17nd0ouPRf8CW22/PuX7L7VvhOPqPuL/DWeOaEZWmrknyxMQE4vF4RcpKJBJwOByw2+0IBAKGt4tGo3C5XLDb7fB4PFllautKKXMlniyo0VUj9oiouHJjr5hKxOYDBw7hM58/hE2bN6ct37R5Mz5z4CA+8/mDlagqUVXVNUn2er0Ih8O4fPnyqsqRJAn9/f3w+/0YGRnB9PS0ocAOBAKIRCLo7e3F2NgYOjs79XWJRAL9/f3o7e3Vyyz3JMSTBTWqasSez+dDa2tr1ouI/qSc2DMSW4Vis1RXlRRuLS2lLbu1tIQPFaXsMolqqa59kuPxOMxmMxwOB9ra2rKe4vfss88aKicajeLYsWN6MA8ODsLhcECW5bxPBpQkCbFYDGNjY7BarQAAt9utrw+Hw/D7/XqZJ06cgMvlgt/vL2vmDZ4sqBFVI/YAwOl04vjx49WtPNE6Vk7sAYVjy0hsluLd3/wKpk2bsGnzFjTvbMH83CxuLd3EO7/5VdllEtVSXVuSU6kU2tracODAAezcuROqqqa9jJIkCfv3/2kAgPbY7EJzLUejUYiiqJ8IVpJlGbIsQxRFfZn2vnK7h2gni8233Y57/p//F5tvux2mTZt4sqB1rdKxp2lubtbL0l5E9CflxB5QOLaMxGYplm7ewK6HHsaj33kO+/6+D49+5znseuhhLN34qCLlE1Xbup/dIplMAgBsNlvackEQ9HW5SJKEYDCYc50sy3oZmWUqZbb8aieLv/ivPbj8h/ew4xP34JX/OYJ3fv1mWeUR1Vs1Yo+Iiis39oqpdGweHYzq/778h/dg/uSn8Nf9X61Y+UTVtu5nt0ilUgCQ1QWiqakJ8/PzObfRkmCLxYJAIAC73Q6Hw6HfgVsslrSyc21bqqODUfx1/1dh/uSnluv7nyeLlScRovWkGrGnmZ6e1gcO+Xy+sm9OiRpRObGnyRdbRmOTaCOpa0vyxMQEzGYzjhw5suqyFEXJOmEsLCzkfK92MtD6HXu9XkSjUXg8Hly4cEFvQZYkSe+PpSgKZFkuegICgOvXr2NmZibv+mvXrhVcT+l4vEpT6+NVydjTykkmkwgGgxBFEYFAAD09PRgbGytaF8ZeZfF4lWYtx54mX2wZjc1cisUdwO9SqXi8jKvmsaprkuz1evH4449DFEXs2LFjVWWlUqmcd9W5aC3FXq9XH/QQDAYRj8cRi8Xg9XrhdDoRDochiiIsFguefPJJANldMHLZunUrdu/enXf9zMxMwfWUjserNKUer9WeXCode93d3eju7tbHBASDQb1Fa+U4gVwYe5XF41WYqqr6A6KA7OOVuT5TLWMPQMHYMhKb+RSLO4DfpVLxeBlXzrEyGnvrfnYLLbAzf46dm5vL6q+l0faTud5ms2FqagoAcPLkSTgcDjgcDgiCAL/fj8nJSbS0tBj7cEQNrlqxl5kIa9skk8miSTJRrbz2b7/G9Q9v4uEj9+ZMhFVVxU9feBNb79yCB//mzyu673JiDygcW9qvpoVik2ijqWuSrM1u0dbWpi8rZVYL4E+zTmT2H5ZlOe/UNdqdd64+x83Nzfq/x8fH9Z+zFEWBoii8SBP9p2rGHtFapqoqrn94E6//eA4AshJlLUF+/cdzuP8vdxZtUS5VObFXDGOTKFtdB+4NDg7imWeegcViwcWLF3Hx4kU0NzfjG9/4RkkzX1itViQSCf3vZDJZNKF1Op2YmJhIWzY9PZ019Y124hgaGoIoipyKimiFSseeoiiIxWJp67SBQ7xBpbXCZDLh4SP34v6/3InXfzyHn77wpt7Ak5kg52tpXq1SY89IbBm9LhJtFHVtSV5YWIDD4cCOHTvgdC4/llmSJHR0dGBsbMxwP2W/3w+Px6MnsQMDA2kJrSzLSCQSaX2q+vr64HK50NLSApvNhqGhIVgsFv0uXDv5WK1WjI6O4syZM4YGDhFtJJWOPVmWEQgEMDs7i0OHDiGVSiEQCMDpdPJCTWuKligD0FuU77HVJkEGSo89LZYKxVax6yLRhqPWUX9/vzowMJC1/LHHHlMfffTRksoaHR1VDx8+rO7duzerzNHRUXXXrl1Z20xMTKgdHR3qrl271Mcee0xNpVJp6w4fPqzu2rVL7enpUWdnZw3X5eLFIx58ewAADyhJREFUi6taT+l4vEpT6vFa7fGtdOzNzs6qjz32mLp37161o6ND/eY3v2m4Loy9yuLxKu7WrVvqK7FL6nf7fqS/XoldUm/dulV021rHnpHYKhSbq/kc/C6VhsfLuHKOldFtTKpaYifgCnrwwQcxNjaGnTt3pi1PJpN49NFH8eqrr9apZqtTbKQlR62WhserNOXMbtEox5exV1k8Xsaoqor/8Q8v63//4/c+Z6gFuVGOr5HP0SiftVZ4vIwrd3YLI9vUtU+yxWLB3Nxc1nJFUfTRu0RERGuV+p99kFda2UeZiCqrWGxVMvbqmiR3dXUhEAjg8uXL+rKZmRk89dRTeh9lIiKitUjNGKT3Od/2nIP5iKgyXvu3XxeMLS0mX/u3X1dkf3V/mIgsy+jo6Eib99HpdOKJJ56oZ9WIiIjyykyQHz5yL954442swXzVHLxHtJGodZh6sa5JMrD8RB+v14vJyUkAyxOXr5w3mYiIaC3JlSBrF+Ncs14wUSZavXyxBVRv6sW6J8nA8lN/OP8wERGtByaTCVvv3JL3YrzyYr71zi1MkIkqpNZTL9YsST5//jympqYMdaP4whe+gG984xu47777alAzIiKi0jz4N39e8Odc7WLOBJmosrIS5R8vL6/G3OQ1G7j33HPPZT1nPp+HHnoIoVCoyjUiIiIqX7GLMRNkoupYmShrqnFTWrMkeW5uDp2dnYbe+/DDD2N6errKNSIiIiKi9aZWUy/WrLuFoihobm429F6LxWK41ZmIiIiINobMQXr32G7ivektVRkoW7OWZEEQ9BksipEkiTNcEBEREZEu3ywWDx+5typzlNcsSe7q6sLQ0FDag0NykWUZkUgEhw4dqlHNiIiIiGgtMzL1YqUT5ZolyV6vFzt27IDL5cJLL72U8z3nz5/HF7/4RezZswdHjx6tVdWIiIiIaA0zOvXi/X+5s2JTL9Z0nuR//ud/xte+9jU89thjsFgssNlsaGpqwsLCAqanp6EoCg4cOIBnnnmmltUiIiIiojWu1lMv1jRJbmpqwsmTJyHLMkZHRzEzM4NkMglBEHDkyBEcOnSIfZGJiIiIKKdaTr1YlyfuCYKA48eP12PXRERERERF1axPMhERERHRelGzJNno9G/VLoOIiIiIqJiaJcmBQADf/va3y94+HA7jqaeeqmCNiIiIiIhyq1mS/OKLL+K3v/0t9u3bh29/+9t44403im4zMzODb33rW9i3bx/m5ubw4osv1qCmRERERLTR1WzgntlsxsmTJyFJEk6dOoVIJAKTyQRBENDU1KQ/snp+fh4LCwuQZRmqqmL//v149tln0d7eXquqEhEREdEGV/PZLURRhCiKAIBEIgFFUTA7OwtZlmGxWLB79260tLTAYrHA6XTWunpERERERPWZAk7T2dlZz90TEREREeXEKeCIiIiIiDI0TJKcSCTgcDhgt9sRCAQMbxeNRuFyuWC32+HxeKpYQ6LGxNgjqo9yY4+IjKlrd4tKkSQJ/f39GBwchCAIGBgYgMfjwfDwcMHtAoEA4vE4Tpw4AavVCkmS9PLyXbRFUSxaLlGtpX5/Fb98Scal167gxrUl/O873kXrg9vxwOcFWD65rWr7rXTsZQqFQjh16hSGh4f1sQxEa8l6ij2fz4dz585lLb906VLWMsYerWW1iruGSJKj0SiOHTum93EeHByEw+GALMsQBCHnNpIkIRaLYWxsDFarFQDgdrsBLCfCY2NjWdsMDAzwZEFrzu+m30MiMoWlJRXqkgoAuHFtCcmJt/DGz95GZ+8efNp2T1X2XenYW0lRFJw5c6Yq9SaqhPUWewDgdDpx/PjxgmUz9mgtq2XcNUR3C0mSsH//fv1vQRAgCELe1ilg+QQjiqJ+kc5ktVrTXrIsAwC8Xm9lK0+0CqnfX0UiMoWbH93STxYadUnFzY9uIRGZQur3V6uy/2rEniYcDnPqR1qz1mPsAUBzc7P+Xu2VibFHa1Wt427dJ8nJZBIAYLPZ0pYLgqCvy0WSJMOzayiKgoGBAZw4caL8ihJVwS9fkrGUcaLItLSk4pfjcsX3Xc3Yk2UZsVgMfX19q68oURWsx9gzgrFHa1mt427dJ8mpVArA8sNKVmpqasL8/HzObbRWYYvFgkAgALvdDofDkfcOfGhoCDabrWjLF1GtXXrtStbddCZ1ScX/ffVKxfddzdgLhUKGWpuJ6mW9xZ5menpaHzDr8/mgKEraesYerWW1jruG6JMMLLf2Zp4wFhYWcr5Xu1CHw2H4/X54vV5Eo1F4PB5cuHAhrRxFUfTBC0Zdv34dMzMzeddfu3at4HpKx+OV341rS4be99G1paodw0rHXjKZxLlz53KOCyiGsVdZPF75rbfY0ySTSQSDQYiiiEAggJ6eHj3Wyo29YnEH8LtUKh6v3Goddw2TJKdSqZx31blYLBYAy/2LtZ99g8Eg4vE4YrFYWr/jWCwGs9lc0oC9rVu3Yvfu3XnXz8zMFFxP6Xi88vvfd7xr6KRx+x2b8x7D1Z5IKh174XAYTqezrJYsxl5l8Xjlt95iDwC6u7vR3d2tX8+CwaD+S44oimXHXrG4A/hdKhWPV26ViDvAeOyt++4W2kU38yejubk5NDc359xGG6iQ2Z/LZrNhamoqbZkkSRzAQGtW64PbYdpsKvge02YTdu3bXvF9VyP2kskkJEkqOvqeqN7WW+wByzM3rWzw0eIxmUwy9mhdqHXcrfskWbvj1fpoaWRZzns3rN15Z24DIOsEkzmCmGgteeDzAjYXOWFs3mzCA478U0KVqxqxd/bsWQCAw+FAa2srWltbAQAejwd2u71idSdarfUWe8Uw9mg9qHXcrfskGVg+YSQSCf3vZDIJRVEKdpFwOp2YmJhIWzY9PZ12gsk3gphorbB8chs6e/dgy+2bsu6uTZtN2HL7JnT27qnaQw0qHXt9fX0YHx/XX1rfyGAwWFYfZaJqWW+xpygKYrFY2jJtwKwoiow9WhdqHXcN0SfZ7/fD4/FAFEX9yUPav4Hlu+tEIpHW17ivrw8ulwstLS2w2WwYGhrC/9/eHcOmccVxHP9HrZouPoYuXS7dzlLw4AFa5VaQQN58Cx2NFGdDLFlhsSeHJRnN4hEyeIyxxApWlcVSjIeOvAxdjyxNpep1sEzDmYt5+IDj8v1IGYDH6e4n/07Pzr27VCo18VCD29/Sv3ZjdmDVftn6SX6v/SqXXSV//vGX/PP3v/LDj9+J89vPsp1f7FO/FtG94MJZEQm9nyuwSuvUPd/3pV6vy3A4lJ2dnfHrL69BpntYB0vtnU6IVquld3d3dSaT0bVa7c5njuPc+U6v19O5XE47jqMrlYr2fX+m793n+vr6QZ9jEnmZMc3rofkuonu3fN/XjuPoXq83077QvWiRl5m4d284HOpKpaIzmYzO5XL66OgodNsm3ZvlOPhZMkNes5snq1m/80hr/fUbzsHYfatSWbVqhrzMmOaVpHzpXrTIy8y32r1ZjiMpx7os5DW7ebKa9TuJuCYZAAAAiBKTZAAAACCASTIAAAAQwCQZAAAACGDh3gJcXl7K48ePV70bwEw+f/4s29vbq96NSNA9rJOkdI/eYd3M2j0myQAAAEAAl1sAAAAAAUySAQAAgAAmyQAAAEAAk2QAAAAggEkyAAAAEMAkeQGazabk83nZ3NyUbDYrnudJp9N58Ngkuy+Hdrstm5ubU//l8/kV7vliBXMpl8v35pLP5+XVq1dTt9fv98XzvInt9fv9ZR3OwtE9M/QuHN0zQ/fM0L1wcere95EcEUREZDQayd7eniil5MWLF+K6rvi+L51ORz58+CDFYnGusUlmmsPp6alYlrWivV2e0WgknueJ7/t3cmk0Gndyef/+/TiXfr8v9XpdLi4u5PT0dDym0+lItVqV58+fy8HBgfi+L61WS5rNpriuu9TjixrdM0PvwtE9M3TPDN0LF8vuaUSmUqnoTCajh8NhpGOTbNYcWq2WdhxH+76/pD1bLdNcgnzf147j6FarNX4vl8vpo6OjqWPXHd0zQ+/C0T0zdM8M3QsXx+5xuUVElFJyfn4uBwcHYtt2ZGOTjBymGwwGcn5+Li9fvpw7F8uyxHVd6fV64/eUUvLkyZOpY9cZ3TNDBuHonhm6Z4YMwsW1e0ySI9JqtcSyrJn+u8hkbJKRw3Ttdlssy5JSqfSg7WxsbMinT5/Gr23blna7/dDdix26Z4YMwtE9M3TPDBmEi2v3mCRH5Pr6Wra2tiIfm2TkMJ1SKpJcLi4uJq65ev36tQwGA8lms1Kv1xOzaIjumSGDcHTPDN0zQwbh4to9JskRUUrJxsZG5GOTbJ4cstnsnVWtSqkF7eFqPPTnQykl5XJZUqmU7O/vj99Pp9PS7Xbl2bNncnZ2JuVyWfL5/NrnR/fM0LtwdM8M3TND98LFtXvc3SIitm1P/Ik/qrFJNk8O01b6Ju3aLtu25ePHjxPvNZtNaTQa49dfruoVuTmRitysDhYRKRQKEyt8v9z2mzdvRORmNXC1WpVyuSzdbjfy41gWumeG3oWje2bonhm6Fy6u3WOSHJGnT5/K27dvIx+bZPPkYNv22i92uY9t23J1dTXxXqlUkmKxKIPBQKrV6p3vnJycjE+cs+bjuq6cnJyI53nS7/fX9lZUdM8MvQtH98zQPTN0L1xcu8flFhHZ2dmR0Wg00w3RTcYmGTlMt7+/L6PRaGKxgWVZYtt26F8Qbk+kpifTdDotIv//Jr6O6J4ZMghH98zQPTNkEC6u3WOSHJF0Oi2FQkFqtdq917qYjE0ycpjOtm0pFArSaDQizWXaifl2EcPtSWMd0T0zZBCO7pmhe2bIIFxcu8flFhE6PDyUarUqnudNPC2m1+vdeQqMydgkI4fpDg8PxfO8iVyUUvLu3bu5tqeUkmq1Kq7rSrFYlFQqJUopaTQaUiqV1v4aN7pnhgzC0T0zdM8MGYSLZfdmeuQIjBwfH+vd3V3tOI7OZDJ6b29PX11dPXhskt2Xw7f49CGtb3LJ5XITuZydnY0/N8llOBzqWq023l4ul9PHx8eL3P2lo3tm6F04umeG7pmhe+Hi1L1HWms91xQdAAAASCiuSQYAAAACmCQDAAAAAUySAQAAgAAmyQAAAEAAk2QAAAAggEkyAAAAEMAkGQAAAAhgkgwAAAAEMEkGAAAAAv4DSckozq168s4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x252 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "params = {'legend.fontsize': 18,\n",
    "         'axes.labelsize': 18,\n",
    "         'axes.titlesize':18,\n",
    "         'xtick.labelsize':18,\n",
    "         'ytick.labelsize':18,\n",
    "         'lines.markersize': 10,\n",
    "         'legend.markerscale': 1.,\n",
    "         'legend.labelspacing': 1.,\n",
    "         'figure.subplot.hspace': 0.3,\n",
    "         'figure.subplot.wspace': 0.05,\n",
    "         }\n",
    "pylab.rcParams.update(params)\n",
    "models = [ 'resnet50', 'efficientnet_b2', 'tf_efficientnet_b5']\n",
    "augs = [ 'central_crop', 'cropflips', 'optimal_policy',]\n",
    "sample_nums = [5, 20]\n",
    "datasets = ['ImageNet']\n",
    "nan = np.nan\n",
    "f = plt.figure(figsize=(10, 3.5))\n",
    "i = 1\n",
    "short_legend_augs_ordered = []\n",
    "for aug in augs:\n",
    "    short_legend_augs_ordered.append(short_legend_augs[aug])\n",
    "\n",
    "for dataset in datasets:\n",
    "    for model in models:\n",
    "        plt.subplot(len(datasets), len(models), i)\n",
    "        i += 1\n",
    "        lines = []\n",
    "        min_ = +150000\n",
    "        max_ = -100500\n",
    "        for j, aug in enumerate(augs):\n",
    "            plot_augs = []\n",
    "            legends = []\n",
    "            for sample_num in sample_nums:\n",
    "                marker = 'o'\n",
    "                if sample_num < 20:\n",
    "                    marker = 'x'\n",
    "                if aug == 'central_crop':\n",
    "                    marker = (5, 1)\n",
    "                if aug != 'central_crop' or sample_num == 20:\n",
    "                    lines.append(plt.plot([j], [metric[dataset][model][aug][sample_num - 1]], marker=marker,\n",
    "                             linewidth=0, c=colors_palette[colors_augs[aug]], alpha=0)[0])\n",
    "                    legends.append(plt.scatter([j], [metric[dataset][model][aug][sample_num - 1]], marker=marker,\n",
    "                                c=colors_palette[colors_augs[aug]]))\n",
    "                    val = metric[dataset][model][aug][sample_num - 1]\n",
    "                    min_ = min(min_, val)\n",
    "                    max_ = max(max_, val)\n",
    "            plt.title(model_names[model] + '\\n' + dataset_names[dataset] + '-C')\n",
    "            plt.xticks(np.arange(len(augs)), short_legend_augs_ordered)\n",
    "            plt.yticks(np.linspace(min_, max_, 5))\n",
    "            from matplotlib.ticker import FormatStrFormatter\n",
    "            plt.gca().yaxis.set_major_formatter(FormatStrFormatter('%.2f'))\n",
    "            if i == 2:\n",
    "                plt.ylabel(r'$mCE$''\\n''(Corruption Error)')\n",
    "cross = plt.scatter([np.nan], [np.nan], color=colors_palette[-1], marker='x',\n",
    "                          label='5 aug. samples')\n",
    "circle = matplotlib.lines.Line2D([], [], color=colors_palette[-1], marker='o', linestyle='None',\n",
    "                          label='20 aug. samples')\n",
    "star = matplotlib.lines.Line2D([], [], color=colors_palette[-1], marker=(5, 1), linestyle='None',\n",
    "                          label='1 aug. sample')\n",
    "f.legend(handles=[star, cross, circle], loc='upper center', ncol=3,\n",
    "        bbox_to_anchor=(0.53, 1.16))\n",
    "plt.tight_layout()\n",
    "plt.savefig('figs/ood_imagenet.pdf', bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_for_table = []\n",
    "models = [ 'resnet50', 'efficientnet_b2', 'tf_efficientnet_b5']\n",
    "augs = [ 'central_crop', 'cropflips', 'optimal_policy',]\n",
    "sample_nums = [5, 10, 20]\n",
    "datasets = ['ImageNet']\n",
    "augs_to_table = {'central_crop': 'cc',\n",
    "                 'cropflips': 'cf',\n",
    "                 'optimal_policy': 'ra'}\n",
    "model_names = {'efficientnet_b2': 'Efficientnet B2',\n",
    "              'resnet50': 'ResNet50',\n",
    "              'tf_efficientnet_b5': 'Efficientnet B5',\n",
    "              'tf_efficientnet_l2_ns_475': 'Efficientnet L2-475'}\n",
    "\n",
    "for dataset in datasets:\n",
    "    for model in models:\n",
    "        for j, aug in enumerate(augs):\n",
    "            for sample_num in sample_nums:\n",
    "                res_for_table.append([dataset, model_names[model], augs_to_table[aug], sample_num, 'mCE', '',\n",
    "                                     round(metric[dataset][model][aug][sample_num - 1], 4), 0., 0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['ImageNet', 'ResNet50', 'cc', 5, 'mCE', '', 0.6871, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'cc', 10, 'mCE', '', 0.6871, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'cc', 20, 'mCE', '', 0.6871, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'cf', 5, 'mCE', '', 0.7225, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'cf', 10, 'mCE', '', 0.708, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'cf', 20, 'mCE', '', 0.701, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'ra', 5, 'mCE', '', 0.6987, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'ra', 10, 'mCE', '', 0.6753, 0.0, 0.0],\n",
       " ['ImageNet', 'ResNet50', 'ra', 20, 'mCE', '', 0.6728, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cc', 5, 'mCE', '', 0.6554, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cc', 10, 'mCE', '', 0.6554, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cc', 20, 'mCE', '', 0.6554, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cf', 5, 'mCE', '', 0.6804, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cf', 10, 'mCE', '', 0.6652, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'cf', 20, 'mCE', '', 0.6579, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'ra', 5, 'mCE', '', 0.6464, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'ra', 10, 'mCE', '', 0.6407, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B2', 'ra', 20, 'mCE', '', 0.6377, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cc', 5, 'mCE', '', 0.559, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cc', 10, 'mCE', '', 0.559, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cc', 20, 'mCE', '', 0.559, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cf', 5, 'mCE', '', 0.609, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cf', 10, 'mCE', '', 0.5935, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'cf', 20, 'mCE', '', 0.5863, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'ra', 5, 'mCE', '', 0.5485, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'ra', 10, 'mCE', '', 0.5437, 0.0, 0.0],\n",
       " ['ImageNet', 'Efficientnet B5', 'ra', 20, 'mCE', '', 0.5376, 0.0, 0.0]]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_for_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}