{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "\n",
    "from scipy.signal import medfilt\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import numpy as np\n",
    "import itertools \n",
    "import pandas as pd\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_learning_average(env_name, ax, experiment, window_size, var = True, OnePlayer = False):\n",
    "    \n",
    "    if OnePlayer:\n",
    "        base_dir = os.getcwd() + '/models_OnePlayer/' + env_name\n",
    "    else:\n",
    "        base_dir = os.getcwd() + '/models_TwoPlayer/' + env_name\n",
    "    results = {}\n",
    "        \n",
    "    for exp in experiment:\n",
    "        results[exp] = {}\n",
    "        for seed in range(5):\n",
    "            try:\n",
    "                with open(base_dir + '/' + exp + str(seed) + '/results', 'rb') as f:\n",
    "                    results[exp][seed] = pickle.load(f)             \n",
    "            except IOError:\n",
    "                print(base_dir + '/' + exp + str(seed) + '/results' + \" not opened\")\n",
    "            \n",
    "    plot_learning_curves(results, ax, env_name, window_size, var, best = False)\n",
    "\n",
    "def plot_learning_best(env_name, seed, ax, experiment, window_size, var = True, OnePlayer = False):\n",
    "\n",
    "    if OnePlayer:\n",
    "        base_dir = os.getcwd() + '/models_OnePlayer/' + env_name\n",
    "    else:\n",
    "        base_dir = os.getcwd() + '/models_TwoPlayer/' + env_name\n",
    "    results = {}\n",
    "        \n",
    "    for i, exp in enumerate(experiment):\n",
    "        results[exp] = {}\n",
    "        try:\n",
    "            with open(base_dir + '/' + exp + str(seed[i]) + '/results', 'rb') as f:\n",
    "                results[exp] = pickle.load(f)             \n",
    "        except IOError:\n",
    "            print(\"No files opened\")\n",
    "            \n",
    "    plot_learning_curves(results, ax, env_name, window_size, var, best = True)\n",
    "    \n",
    "def plot_learning_curves(results, ax, env_name, window_size, var, best):    \n",
    "    colors = ['#396ab1', '#cc2529', '#3e9651', '#396ab1', '#da7c30', '#94823d', '#535154', '#006400', '#00FF00', '#800000', '#F08080', '#FFFF00', '#000000', '#C0C0C0']\n",
    "\n",
    "    idx = 0\n",
    "    final_avg = []\n",
    "    final_std = []\n",
    "    for exp in results:\n",
    "        if(best):\n",
    "            merged = list(itertools.chain(*(results[exp]['eval_rewards'])))\n",
    "        else:\n",
    "            reward = []    \n",
    "            for seed in results[exp]:\n",
    "                reward.append(list(itertools.chain(*(results[exp][seed]['eval_rewards']))))\n",
    "            merged = list(itertools.chain.from_iterable(reward))\n",
    "        result_dict = {}\n",
    "        for i in range(0, len(merged), 2):\n",
    "            if(math.isnan(merged[i+1]) == False):\n",
    "                if str(merged[i]) in result_dict.keys():\n",
    "                    result_dict[str(merged[i])].append(merged[i+1]) \n",
    "                else:\n",
    "                    result_dict[str(merged[i])] = [merged[i+1]]\n",
    "        length = len(result_dict)\n",
    "        x = np.zeros(length)\n",
    "        y_avg = np.zeros(length)\n",
    "        y_std = np.zeros(length)\n",
    "        for i, (key, value) in enumerate(result_dict.items()):\n",
    "            x[i] = key\n",
    "            y_avg[i] = np.mean(value)\n",
    "            y_std[i] = np.std(value)\n",
    "\n",
    "        if 'RMSprop' in exp:\n",
    "            l = 'Baseline'\n",
    "        else:\n",
    "            l = 'SGLD'\n",
    "        ax.plot(x, y_avg, color = colors[idx], label = exp)\n",
    "#         ax.plot(x, y_avg, color = colors[idx], label = l)\n",
    "        if (var):\n",
    "            ax.fill_between(x, (y_avg)-(y_std), (y_avg)+(y_std), facecolor=colors[idx], alpha=0.4, interpolate=True)\n",
    "        idx += 1  \n",
    "      \n",
    "    ax.set_xlabel('Timesteps')\n",
    "    ax.set_ylabel('Reward')\n",
    "    ax.ticklabel_format(style='scientific', axis='x', scilimits=(0,0))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_learning_all(experiment, title, comparison, OnePlayer, best, seed=None):\n",
    "    cols = 1\n",
    "    rows = 1\n",
    "    envs = ['simple-v0']\n",
    "    fill = [True, False]\n",
    "    for f in fill:\n",
    "        fig, axs = plt.subplots(rows, cols, figsize=(20*cols/3, 9*rows/2))\n",
    "        for i, env in enumerate(envs):\n",
    "#             ax = axs.flatten()[i]\n",
    "            ax = axs\n",
    "            if(comparison):\n",
    "                exp = experiment[i]\n",
    "            else:\n",
    "                exp = experiment\n",
    "            if(best):\n",
    "                plot_learning_best(env, seed[i], ax, exp, window_size = 1, var = f, OnePlayer = OnePlayer)\n",
    "            else:\n",
    "                plot_learning_average(env, ax, exp, window_size = 1, var = f, OnePlayer = OnePlayer)\n",
    "            if(comparison):\n",
    "                ax.legend()\n",
    "            else:\n",
    "                if i == 0:\n",
    "                    ax.legend()\n",
    "        plt.tight_layout()\n",
    "        \n",
    "        if(OnePlayer):\n",
    "            folder = 'OnePlayer'\n",
    "        else:\n",
    "            folder = 'TwoPlayer'\n",
    "            \n",
    "        if(f):\n",
    "            plt.savefig('plots/'+ folder + '/Learning_curve/Std/' + title +'.pdf')                    \n",
    "        else:\n",
    "            plt.savefig('plots/'+ folder + '/Learning_curve/' + title +'.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ythua\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\matplotlib\\pyplot.py:522: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 93264 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52212 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 92044 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52596 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 93672 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43376 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 93220 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 58872 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 96484 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54612 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 92136 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43844 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108988 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 57084 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 106012 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54188 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 94492 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43232 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 88568 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52320 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 85572 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43256 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 83160 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44676 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 94740 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 55072 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108536 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54016 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 102020 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48188 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 96348 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 55712 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 103744 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52932 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 101944 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44420 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for f in [True, False]:\n",
    "    for delta in ['0.0','0.1','0.2']:\n",
    "        simple_exp = ['SGLD_thermal_0.001/delta_'+ delta +'/lr_0.001/', 'SGLD_thermal_0.0001/delta_'+ delta +'/lr_0.001/', 'SGLD_thermal_1e-05/delta_'+ delta +'/lr_0.001/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'SGLD_rho_'+ delta + '_lr_0.001', comparison = True, OnePlayer = f, best = False)\n",
    "        simple_exp = ['SGLD_thermal_0.001/delta_'+ delta +'/lr_0.0001/', 'SGLD_thermal_0.0001/delta_'+ delta +'/lr_0.0001/', 'SGLD_thermal_1e-05/delta_'+ delta +'/lr_0.0001/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'SGLD_rho_'+ delta + '_lr_0.0001', comparison = True, OnePlayer = f, best = False)\n",
    "        simple_exp = ['SGLD_thermal_0.001/delta_'+ delta +'/lr_1e-05/', 'SGLD_thermal_0.0001/delta_'+ delta +'/lr_1e-05/', 'SGLD_thermal_1e-05/delta_'+ delta +'/lr_1e-05/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'SGLD_rho_'+ delta + '_lr_1e-05', comparison = True, OnePlayer = f, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ythua\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\matplotlib\\pyplot.py:522: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ythua\\policy-gradient-reinforce/models_TwoPlayer/simple-v0/E_SGLD_thermal_0.001/delta_0.2/lr_0.001/4/results not opened\n",
      "C:\\Users\\ythua\\policy-gradient-reinforce/models_TwoPlayer/simple-v0/E_SGLD_thermal_0.001/delta_0.2/lr_0.001/4/results not opened\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 103408 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53004 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 94316 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47888 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 85328 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 39660 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 115664 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 59280 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 96064 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 41908 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 101288 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45344 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 112368 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 56244 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 109864 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45224 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108584 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47800 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 104884 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52280 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 84908 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 40460 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 83592 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38908 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 110368 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 55732 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 103944 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44032 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108244 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46688 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 111744 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 56528 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 104872 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44268 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108924 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45636 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for f in [True, False]:\n",
    "    for delta in ['0.0','0.1','0.2']:\n",
    "        simple_exp = ['E_SGLD_thermal_0.001/delta_'+ delta +'/lr_0.001/', 'E_SGLD_thermal_0.0001/delta_'+ delta +'/lr_0.001/', 'E_SGLD_thermal_1e-05/delta_'+ delta +'/lr_0.001/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'E_SGLD_rho_'+ delta + '_lr_0.001', comparison = True, OnePlayer = f, best = False)\n",
    "        simple_exp = ['E_SGLD_thermal_0.001/delta_'+ delta +'/lr_0.0001/', 'E_SGLD_thermal_0.0001/delta_'+ delta +'/lr_0.0001/', 'E_SGLD_thermal_1e-05/delta_'+ delta +'/lr_0.0001/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'E_SGLD_rho_'+ delta + '_lr_0.0001', comparison = True, OnePlayer = f, best = False)\n",
    "        simple_exp = ['E_SGLD_thermal_0.001/delta_'+ delta +'/lr_1e-05/', 'E_SGLD_thermal_0.0001/delta_'+ delta +'/lr_1e-05/', 'E_SGLD_thermal_1e-05/delta_'+ delta +'/lr_1e-05/']\n",
    "        exp = [simple_exp]    \n",
    "        plot_learning_all(exp, 'E_SGLD_rho_'+ delta + '_lr_1e-05', comparison = True, OnePlayer = f, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 110644 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54328 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 108980 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53176 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 116128 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47196 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 111544 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50284 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 107976 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 51116 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 112488 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47988 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for f in [True, False]:\n",
    "    simple_exp = ['RMSprop/delta_0.0/lr_0.001/', 'RMSprop/delta_0.0/lr_0.0001/', 'RMSprop/delta_0.0/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'RMSprop_rho_0.0', comparison = True, OnePlayer = f, best = False)\n",
    "    simple_exp = ['RMSprop/delta_0.1/lr_0.001/', 'RMSprop/delta_0.1/lr_0.0001/', 'RMSprop/delta_0.1/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'RMSprop_rho_0.1', comparison = True, OnePlayer = f, best = False)\n",
    "    simple_exp = ['RMSprop/delta_0.2/lr_0.001/', 'RMSprop/delta_0.2/lr_0.0001/', 'RMSprop/delta_0.2/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'RMSprop_rho_0.2', comparison = True, OnePlayer = f, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 101180 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52720 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 109760 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52468 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 104020 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 51032 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 109636 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48808 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 111800 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48712 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 120064 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45700 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for f in [True, False]:\n",
    "    simple_exp = ['ExtraAdam/delta_0.0/lr_0.001/', 'ExtraAdam/delta_0.0/lr_0.0001/', 'ExtraAdam/delta_0.0/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'ExtraAdam_rho_0.0', comparison = True, OnePlayer = f, best = False)\n",
    "    simple_exp = ['ExtraAdam/delta_0.1/lr_0.001/', 'ExtraAdam/delta_0.1/lr_0.0001/', 'ExtraAdam/delta_0.1/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'ExtraAdam_rho_0.1', comparison = True, OnePlayer = f, best = False)\n",
    "    simple_exp = ['ExtraAdam/delta_0.2/lr_0.001/', 'ExtraAdam/delta_0.2/lr_0.0001/', 'ExtraAdam/delta_0.2/lr_1e-05/']\n",
    "    exp = [simple_exp]    \n",
    "    plot_learning_all(exp, 'ExtraAdam_rho_0.2', comparison = True, OnePlayer = f, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## One-Player Parameter\n",
    "delta_0.0\n",
    "ExtraAdam_lr_0.001\n",
    "RMSprop_lr_0.001\n",
    "SGLD_thermal_1e-4_lr_1e-4\n",
    "\n",
    "delta_0.1\n",
    "ExtraAdam_lr_0.001\n",
    "RMSprop_lr_0.0001\n",
    "SGLD_thermal_1e-4_lr_1e-4\n",
    "\n",
    "delta_0.2\n",
    "ExtraAdam_lr_0.001\n",
    "RMSprop_lr_0.0001\n",
    "SGLD_thermal_1e-4_lr_1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Two-Player Parameter\n",
    "delta_0.0\n",
    "ExtraAdam_lr_0.001\n",
    "RMSprop_lr_0.001\n",
    "SGLD_thermal_1e-4_lr_1e-4\n",
    "\n",
    "delta_0.1\n",
    "ExtraAdam_lr_0.001\n",
    "RMSprop_lr_0.001\n",
    "SGLD_thermal_1e-3_lr_1e-4\n",
    "\n",
    "delta_0.2\n",
    "ExtraAdam_lr_0.0001\n",
    "RMSprop_lr_0.0001\n",
    "SGLD_thermal_1e-5_lr_1e-3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mass Uncertainty Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_env(env_name, params, ax, var, OnePlayer, best, seed=None):\n",
    "\n",
    "    if OnePlayer:\n",
    "        base_dir = os.getcwd() + '/models_OnePlayer/' + env_name\n",
    "    else:\n",
    "        base_dir = os.getcwd() + '/models_TwoPlayer/' + env_name\n",
    "    results = {}\n",
    "        \n",
    "    for i, exp in enumerate(params):\n",
    "        if(best):\n",
    "            upper = seed[i] + 1\n",
    "            lower = seed[i]\n",
    "        else:\n",
    "            upper = 5\n",
    "            lower = 0\n",
    "        results[exp] = {}\n",
    "        for s in range(lower, upper):\n",
    "            try:\n",
    "                with open(base_dir + '/' + exp + str(s) + '/results_model', 'rb') as f:\n",
    "                    results[exp][s] = pickle.load(f)                         \n",
    "            except IOError:\n",
    "                print(\"No file opened\")\n",
    "            \n",
    "    plot_model(results, ax, env_name, var, OnePlayer)\n",
    "\n",
    "def plot_model(results, ax, env_name, var, OnePlayer):\n",
    "    x = [i for i in range(20)]\n",
    "    colors = ['#396ab1','#3e9651', '#cc2529', '#da7c30', '#94823d', '#535154', '#00FF00', '#800000', '#F08080', '#FFFF00', '#000000', '#C0C0C0']\n",
    "    \n",
    "    idx = 0\n",
    "    for exp in results:\n",
    "        if exp not in ['results_model']:\n",
    "            avg = []\n",
    "            std = []\n",
    "            for seed in results[exp]:\n",
    "                avg_seed = []\n",
    "                std_seed = []\n",
    "                for mass in results[exp][seed]:\n",
    "                    avg_seed.append(np.mean(results[exp][seed][mass]))\n",
    "                    std_seed.append(np.std(results[exp][seed][mass]) / np.sqrt(len(results[exp][seed][mass]) - 1) * 2)\n",
    "                avg.append(avg_seed)\n",
    "                std.append(std_seed)\n",
    "            avg = np.array(avg)\n",
    "            std = np.array(std)\n",
    "            avg = np.mean(avg, axis=0)\n",
    "            std = np.mean(std, axis=0)\n",
    "            \n",
    "        if 'RMSprop' in exp:\n",
    "            l = 'GAD (RMSprop)'\n",
    "        elif 'ExtraAdam' in exp:\n",
    "            l = 'Extra-Adam'\n",
    "        else:\n",
    "            l = 'MixedNE-LD (RMSProp)'        \n",
    "        if(OnePlayer):\n",
    "            l = r'$\\beta=$'+ str(exp[45:48]) + r', $K_t$='+ str(exp[52:53])\n",
    "            ax.plot(x, avg, label=l, color=colors[idx], linestyle=\"--\")\n",
    "        else:\n",
    "            ax.plot(x, avg, label=l, color=colors[idx], linestyle=\"-\")\n",
    "        if(var):\n",
    "            ax.fill_between(x, np.array(avg) - np.array(std), np.array(avg) + np.array(std), facecolor=colors[idx], alpha=0.2, interpolate=True)\n",
    "        idx += 1\n",
    "  \n",
    "    ax.set_xticks([0, 4.75, 9.5, 14.25, 19])\n",
    "    ax.set_xticklabels([0.0, 0.1, 0.2, 0.3, 0.4])\n",
    "    ax.set_xlabel(r'Environment parameter $\\rho$')\n",
    "    ax.set_ylabel('Reward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_env_all(params, title, comparison, OnePlayer, best, seed = None):\n",
    "    cols = 1\n",
    "    rows = 1\n",
    "    envs = ['simple-v0']\n",
    "    fill = [True, False]\n",
    "    for f in fill:\n",
    "        fig, axs = plt.subplots(rows, cols, figsize=(20*cols/3, 9*rows/2))\n",
    "        for i, env in enumerate(envs):\n",
    "#             ax = axs.flatten()[i]\n",
    "            ax = axs\n",
    "            if(comparison):\n",
    "                param = params[i]\n",
    "            else:\n",
    "                param = params\n",
    "            if(best):\n",
    "                plot_env(env, param, ax, var = f, OnePlayer = OnePlayer, best = best, seed = seed[i])\n",
    "            else:\n",
    "                plot_env(env, param, ax, var = f, OnePlayer = OnePlayer, best = best)                \n",
    "            if(comparison):\n",
    "                ax.legend(loc='upper right')\n",
    "            else:\n",
    "                if i == 0:\n",
    "                    ax.legend()\n",
    "        plt.tight_layout()\n",
    "        \n",
    "        if(OnePlayer):\n",
    "            folder = 'OnePlayer'\n",
    "        else:\n",
    "            folder = 'TwoPlayer'\n",
    "            \n",
    "        if(f):\n",
    "            if(best):\n",
    "                plt.savefig('plots/'+ folder + '/Mass_Uncertainty/' + title +'_best_fill.pdf')\n",
    "            else:\n",
    "                plt.savefig('plots/'+ folder + '/Mass_Uncertainty/' + title +'_fill.pdf')                    \n",
    "        else:\n",
    "            if(best):\n",
    "                plt.savefig('plots/'+ folder + '/Mass_Uncertainty/' + title +'_best.pdf')\n",
    "            else:\n",
    "                plt.savefig('plots/'+ folder + '/Mass_Uncertainty/' + title +'.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_env_merge(params, title, comparison):\n",
    "    cols = 1\n",
    "    rows = 1\n",
    "    envs = ['simple-v0']\n",
    "    fill = [True, False]\n",
    "    for f in fill:\n",
    "        fig, axs = plt.subplots(rows, cols, figsize=(20*cols/3, 9*rows/2))\n",
    "        for i, env in enumerate(envs):\n",
    "            ax = axs\n",
    "            if(comparison):\n",
    "                param = params[i]\n",
    "            else:\n",
    "                param = params\n",
    "            plot_env(env, param[0:3], ax, var = f, OnePlayer = True, best = False)\n",
    "            plot_env(env, param[3:6], ax, var = f, OnePlayer = False, best = False)                \n",
    "            if(comparison):\n",
    "                ax.legend(loc='upper right')\n",
    "            else:\n",
    "                if i == 0:\n",
    "                    ax.legend()\n",
    "        plt.tight_layout()\n",
    "        if(f):\n",
    "            plt.savefig('plots/Merged/Mass_Uncertainty/'+ title +'_fill.pdf')\n",
    "        else:\n",
    "            plt.savefig('plots/Merged/Mass_Uncertainty/'+ title +'.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merged Plots for OnePlayer and TwoPlayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 53316 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 40068 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 51864 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38244 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48228 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 37216 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "simple_exp = ['RMSprop/delta_0.0/lr_0.001/', 'ExtraAdam/delta_0.0/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.0/lr_0.0001/',\\\n",
    "              'RMSprop/delta_0.0/lr_0.001/', 'ExtraAdam/delta_0.0/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.0/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_merge(exp, title = 'rho_0.0', comparison = True)\n",
    "simple_exp = ['RMSprop/delta_0.1/lr_0.0001/', 'ExtraAdam/delta_0.1/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.1/lr_0.0001/',\\\n",
    "              'RMSprop/delta_0.1/lr_0.001/', 'ExtraAdam/delta_0.1/lr_0.001/', 'SGLD_thermal_0.001/delta_0.1/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_merge(exp, title = 'rho_0.1', comparison = True)\n",
    "simple_exp = ['RMSprop/delta_0.2/lr_0.0001/', 'ExtraAdam/delta_0.2/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.2/lr_0.0001/',\\\n",
    "              'RMSprop/delta_0.2/lr_0.0001/', 'ExtraAdam/delta_0.2/lr_0.0001/', 'E_SGLD_thermal_1e-05/delta_0.2/lr_0.001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_merge(exp, title = 'rho_0.2', comparison = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 36956 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 29968 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 36544 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 28344 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 35092 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 28384 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## One-Player Parameter\n",
    "simple_exp = ['RMSprop/delta_0.0/lr_0.001/', 'ExtraAdam/delta_0.0/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.0/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.0', comparison = True, OnePlayer = True, best = False)\n",
    "simple_exp = ['RMSprop/delta_0.1/lr_0.0001/', 'ExtraAdam/delta_0.1/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.1/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.1', comparison = True, OnePlayer = True, best = False)\n",
    "simple_exp = ['RMSprop/delta_0.2/lr_0.0001/', 'ExtraAdam/delta_0.2/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.2/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.2', comparison = True, OnePlayer = True, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46132 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 33660 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 39308 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 29612 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34444 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 26280 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Two-Player Parameter\n",
    "simple_exp = ['RMSprop/delta_0.0/lr_0.001/', 'ExtraAdam/delta_0.0/lr_0.001/', 'SGLD_thermal_0.0001/delta_0.0/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.0', comparison = True, OnePlayer = False, best = False)\n",
    "simple_exp = ['RMSprop/delta_0.1/lr_0.001/', 'ExtraAdam/delta_0.1/lr_0.001/', 'SGLD_thermal_0.001/delta_0.1/lr_0.0001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.1', comparison = True, OnePlayer = False, best = False)\n",
    "simple_exp = ['RMSprop/delta_0.2/lr_0.0001/', 'ExtraAdam/delta_0.2/lr_0.0001/', 'E_SGLD_thermal_1e-05/delta_0.2/lr_0.001/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'rho_0.2', comparison = True, OnePlayer = False, best = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ablation study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 76524 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 58036 characters]\n",
      "text/plain": [
       "<Figure size 480x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "param = 'SGLD_thermal_0.0001/delta_0.2/lr_0.0001/'\n",
    "simple_exp = [param+'beta_0.5/Kt_1/', param+'beta_0.5/Kt_2/', param+'beta_0.5/Kt_5/', param+'beta_0.9/Kt_1/', param+'beta_0.9/Kt_2/',\\\n",
    "              param+'beta_0.9/Kt_5/', param+'beta_1.0/Kt_1/', param+'beta_1.0/Kt_2/', param+'beta_1.0/Kt_5/']\n",
    "exp = [simple_exp]    \n",
    "plot_env_all(exp, title = 'Ablation', comparison = True, OnePlayer = True, best = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
