diff --git a/mdp.ipynb b/mdp.ipynb index 909b874ca..ece50c2d6 100644 --- a/mdp.ipynb +++ b/mdp.ipynb @@ -17,7 +17,7 @@ }, "outputs": [], "source": [ - "from mdp import MDP, GridMDP, sequential_decision_environment, value_iteration" + "from mdp import *" ] }, { @@ -230,7 +230,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -283,17 +283,7 @@ { "data": { "text/plain": [ - "{(0, 0): 0.2962883154554812,\n", - " (0, 1): 0.3984432178350045,\n", - " (0, 2): 0.5093943765842497,\n", - " (1, 0): 0.25386699846479516,\n", - " (1, 2): 0.649585681261095,\n", - " (2, 0): 0.3447542300124158,\n", - " (2, 1): 0.48644001739269643,\n", - " (2, 2): 0.7953620878466678,\n", - " (3, 0): 0.12987274656746342,\n", - " (3, 1): -1.0,\n", - " (3, 2): 1.0}" + "{(0, 0): 0.2962883154554812,\n (0, 1): 0.3984432178350045,\n (0, 2): 0.5093943765842497,\n (1, 0): 0.25386699846479516,\n (1, 2): 0.649585681261095,\n (2, 0): 0.3447542300124158,\n (2, 1): 0.48644001739269643,\n (2, 2): 0.7953620878466678,\n (3, 0): 0.12987274656746342,\n (3, 1): -1.0,\n (3, 2): 1.0}" ] }, "execution_count": 9, @@ -397,8 +387,7 @@ " slider.value = i\n", " time.sleep(float(time_step))\n", " \n", - " return visualize_callback\n", - " " + " return visualize_callback" ] }, { @@ -411,8 +400,7 @@ "source": [ "columns = 4\n", "rows = 3\n", - "U_over_time = value_iteration_instru(sequential_decision_environment)\n", - " " + "U_over_time = value_iteration_instru(sequential_decision_environment)" ] }, { @@ -434,15 +422,43 @@ "scrolled": true }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Widget Javascript not detected. It may not be installed or enabled properly.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Widget Javascript not detected. It may not be installed or enabled properly.\n" + ] + }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAADtCAYAAAAr+2lCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADM5JREFUeJzt2lFolGe+gPFn0ggH1pKEHPWQ0a2Cya7scpz1ECxyEETY\ngANGUKgNbEqoopbdhFKkXikKB9obRXSDVsqxWch2KdQG9cRVKAgKktYajAtdrWldndhIUxs3vRGZ\nOReJaULSONvqzPjv87txJu/7hTd/Ph8+JyZyuRySFFFZsQ8gSU+KgZMUloGTFJaBkxSWgZMUloGT\nFFb5TIsjI/h/SKQimf1sothHeHrkctMOyyc4SWEZOElhGThJYRk4SWEZOElhGThJYRk4SWEZOElh\nGThJYRk4SWEZOElhGThJYRk4SWEZOElhGThJYRk4SWEZOElhGThJYRk4SWEZOElhGThJYRk4SWEZ\nOElhGThJYRk4SWEZOElhGThJYRk4SWEZOElhGThJYRk4SWGVXOC2b28llaplxYoUly/3Trvnxo0v\nWLXqeVKpOlpaXuTBgweT1i9e/Iiqqll0db1fiCMXhXPKn7N6tJeBecB/zrCnFagFUsDEKZ4CfgnU\nAW8+qQP+QCUVuNOnu+nvv05v7zX27z9MW9vWafft3Pk6ra2v0dt7lYqKSjo63h5fy2az7Nq1g9Wr\nGwp17IJzTvlzVvlpAf46w3o3cB24BhwGHk4xC/x+7Nq/AX8GPn1yx/yXlVTgTp7soqmpGYD6+uXc\nuzfMnTuDU/adPfshjY3rAWhqeonjx4+Nrx06dIB16zYwZ87cwhy6CJxT/pxVfv4bqJphvQtoHnu9\nHBgGBoEeRp/qngNmARvH9paKkgrcwECGZHLB+PuamiQDA5lJe4aGhqisrKKsbPToyeR8bt8eGL/+\nxIkP2LRpG7lcrnAHLzDnlD9n9XhkgAUT3s8f+9r3fb1UlFTgfqwdO15lz56JnwL8dG/ImTin/Dmr\n6T0tUygv9gGOHGnn6NEjJBIJli2rJ5O5Ob6WydyipiY5aX91dTXDw9+QzWYpKyubtOfSpY9padlI\nLpdjaOgrzpzpprx8Fun02oL+TE+Cc8qfs3r8ksDNCe9vjX3tPvCPab5eKor+BLd58yucP3+Jc+c+\nIZ1upLOzA4CengtUVFQyd+68KdesXLmKY8feA6Cz8x3S6UYA+vr66evr58qVz2ls3MDeve1hbkTn\nlD9n9cPk+P4ns7VAx9jrC0Alo791rQc+A24wGrt3x/aWiqIHbqKGhjUsXLiIpUsX09a2hX372sfX\n1q9PMzj4JQC7d7/BwYN7SaXquHv3a5qbX57yvRKJRMHOXWjOKX/OKj9NwArgKvBz4H8Z/W3pW2Pr\na4BFwGJgC/Bwis8AB4HfAr9i9JcMSwp26kdLzPTB6cjIU/NPbSmc2c/GDepjl8tNO6ySeoKTpMfJ\nwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJCsvA\nSQrLwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJCsvASQrLwEkKy8BJ\nCsvASQrLwEkKy8BJCqu82AeIYvbPcsU+wlNh5NtEsY/w1EjgPZWv75uUT3CSwjJwksIycJLCMnCS\nwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLC\nMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIq\nucBt395KKlXLihUpLl/unXbPjRtfsGrV86RSdbS0vMiDBw8mrV+8+BFVVbPo6nq/EEcuuFOnTvHL\nJUuo+8UvePPNN6fd09raSm1dHanf/Ibe3t5/6dpovKfy8XdgBfBvwN4Z9n0BPA/UAS8CE+fUCtQC\nKWD6ORdaSQXu9Olu+vuv09t7jf37D9PWtnXafTt3vk5r62v09l6loqKSjo63x9ey2Sy7du1g9eqG\nQh27oLLZLL//wx/466lT/O3KFf787rt8+umnk/Z0d3dzvb+fa1evcvjQIbZu25b3tdF4T+WrGjgA\nbH/EvteB14CrQCXwcE7dwHXgGnAYmH7OhVZSgTt5soumpmYA6uuXc+/eMHfuDE7Zd/bshzQ2rgeg\nqekljh8/Nr526NAB1q3bwJw5cwtz6ALr6emhtraW5557jlmzZrHxhRfo6uqatKerq4vm3/0OgOXL\nlzM8PMzg4GBe10bjPZWvfwf+Cyh/xL4PgfVjr18CPhh73QU0j71eDgwDU+dcaCUVuIGBDMnkgvH3\nNTVJBgYyk/YMDQ1RWVlFWdno0ZPJ+dy+PTB+/YkTH7Bp0zZyuVzhDl5AmUyGBfPnj7+fP38+mczk\nGWUGBliwYMGUPflcG4331OM0BFTxXTbmAw9nmQEWTNibnLBWPCUVuB9rx45X2bNn4udKP/UbcpR/\nMX8476mn26OeR5+4I0faOXr0CIlEgmXL6slkbo6vZTK3qKlJTtpfXV3N8PA3ZLNZysrKJu25dOlj\nWlo2ksvlGBr6ijNnuikvn0U6vbagP9OTlEwm+cfN72Z069YtksnJM0rW1HBzmj33799/5LUReE/l\nqx04AiSA/wP+4xH7q4FvgCyjz0a3GH1SY+zPmxP2TlwrnqI/wW3e/Arnz1/i3LlPSKcb6ezsAKCn\n5wIVFZXMnTtvyjUrV67i2LH3AOjsfId0uhGAvr5++vr6uXLlcxobN7B3b3uQG/E79fX1fPbZZ9y4\ncYP79+/z7l/+wtq1k3/GtWvX0vGnPwFw4cIFKisrmTdvXl7XRuA9la9XgEvAJ0yO20xPqauA98Ze\nvwM0jr1eC3SMvb7A6C8gps650IoeuIkaGtawcOEili5dTFvbFvbtax9fW78+zeDglwDs3v0GBw/u\nJZWq4+7dr2lufnnK90okEgU7dyE988wzHDxwgN82NPCrX/+ajS+8wJIlSzh8+DBvvfUWAGvWrGHR\nwoUsrq1ly9attP/xjzNeG5n3VL4GGf0MbR/wP8DPgZGxtTTw5djrNxj9byR1wNfAwzmtARYBi4Et\njD4dFl9ips9nRkb8wCFfs3/mqPIx8m3kSDxezz5b7BM8PXI5pr2xSuoJTpIeJwMnKSwDJyksAycp\nLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKSwDJyks\nAycpLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKSwDJyksAycpLAMnKazy\nYh8gipFvE8U+goL55z+LfYKnn09wksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJw\nksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCMnCS\nwjJwksIycJLCMnCSwjJwksIycJLCMnCSwjJwksIycJLCKrnAbd/eSipVy4oVKS5f7p12z40bX7Bq\n1fOkUnW0tLzIgwcPJq1fvPgRVVWz6Op6vxBHLgrnlD9nlZ+IcyqpwJ0+3U1//3V6e6+xf/9h2tq2\nTrtv587XaW19jd7eq1RUVNLR8fb4WjabZdeuHaxe3VCoYxecc8qfs8pP1DmVVOBOnuyiqakZgPr6\n5dy7N8ydO4NT9p09+yGNjesBaGp6iePHj42vHTp0gHXrNjBnztzCHLoInFP+nFV+os6ppAI3MJAh\nmVww/r6mJsnAQGbSnqGhISorqygrGz16Mjmf27cHxq8/ceIDNm3aRi6XK9zBC8w55c9Z5SfqnEoq\ncD/Wjh2vsmfPmxO+UjqDLiXOKX/OKj+lOqfyYh/gyJF2jh49QiKRYNmyejKZm+NrmcwtamqSk/ZX\nV1czPPwN2WyWsrKySXsuXfqYlpaN5HI5hoa+4syZbsrLZ5FOry3oz/QkOKf8Oav8/BTmVPQnuM2b\nX+H8+UucO/cJ6XQjnZ0dAPT0XKCiopK5c+dNuWblylUcO/YeAJ2d75BONwLQ19dPX18/V658TmPj\nBvbubS/6gB8X55Q/Z5Wfn8Kcih64iRoa1rBw4SKWLl1MW9sW9u1rH19bvz7N4OCXAOze/QYHD+4l\nlarj7t2vaW5+ecr3SiQSBTt3oTmn/Dmr/ESdU2KmDwRHRkrkH9KSNIPZs5m2qiX1BCdJj5OBkxSW\ngZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGTFJaB\nkxSWgZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGTFJaBkxSWgZMUloGT\nFJaBkxSWgZMUViKXyxX7DJL0RPgEJyksAycpLAMnKSwDJyksAycpLAMnKaz/B9v3wubCyTXSAAAA\nAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbEAAAFKCAYAAACThWFrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAFNxJREFUeJzt3GFslfW9wPHvESmjlSbjQk8B9SKsE8mkIAgcIhGwG3NM\nEF84cCZyjbq5ORlsigtMQcwGW0CHyEInRnMnYwXDNNvYZJiGGyggaJHEuJt5xw1609NhrnS1jnKb\nc1+0Fkp7aKdgz6/9fpK+6N//c/ocfnn67dOeYyKTySBJUkQXdfcJSJL0cRkxSVJYRkySFJYRkySF\nZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhXdzVjfX1+L/2kNSjXDIg0d2noHPJZDodkHdikqSwjJgk\nKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgk\nKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgk\nKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgk\nKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLCMmKSpLB6TMQee+xhSkqGUlSUz+zZX+Ttt//S\n6THbt29l/PirGDy4P6lUKS+/vCPr3jVrVlFYeBEPPbT4fJ52r+F8cpezyT3/AcwGhtH8TfqlLhxT\nCYwHPgN8Hniugz1bgauA/kApkH1qcfSIiK1du5ry8vWsW1dOZeUB8vMLmDt3Jo2NjVmP2bdvL3fe\neRsLFtzNnj3VzJo1h/nzb+att95st/fQoVd59tlyrr669EI+jR7L+eQuZ5ObPgDGAhuARBf2HwW+\nCtwAHAYWAncBO8/Ysxe4DbgbqAbmADcD7acWSyKTyXRpY309XdvYDUpKhrJw4QPcd98iAOrq6hg5\nMsnGjc9xyy23dnjMggXzaGhooKLi9M84M2akKC0dx+OPb2hdq6+vZ+rU8TzxxM9ZvXolY8aMY9Wq\ntRf2CfUwzid39fbZXDKgK4noXhcBv6H5ziybJTTfVb1xxtp84ATw+5bP5wENtL2rSwHjaI5lTspk\nOh1Q+Duxo0f/Sjpdw7RpN7SuFRYWMmHCJA4cqMp63IEDVUyfXtZmraxsZrtjFi/+NjfeeBPXXz/j\n/J54L+F8cpez6Tn2AWVnrc0EzpxIVRf2RHRxd5/AJ5VO15BIJCgqSrZZLypKkk7XnPO4zo7Ztm0L\nR45Us3v3wfN70r2I88ldzqbnqAGSZ60lgTrgJNDvHHuyTzqGcHdiFRWbGTJkAEOGDGDo0EJOnTp1\nQb7OO+8cY8mS77Jp0/P07dv3gnyNnsj55C5no54o3J3YrFlzuPbaya2fnzz5DzKZDLW16TY/HdbW\npiktHZf1cZLJYmpr023WamvTJJPFAFRXv8bx43/juuuu4aO/GzY1NbFnz27Ky9fz3nsnSSRy//fp\nnzbnk7ucTc9VDKTPWksDhTTfhZ1rT/GFPbULLtydWEFBAVdcMaL1Y9So0SSTxVRW7mrdU1dXx8GD\n+5k0aUrWx5k4MdXmGIBXXtnJxIkpAKZPL2P//iPs3VtNVdVhqqoOc801E/ja125n797DXoRZOJ/c\n5Wx6rhSw66y1l1vWz7Vn51l7Igp3J9aRb33ru/zkJ48xYsTn+Nd/Hc7KlT9k2LBLmTVrTuuee+65\ng6FDh7F8+Y8AuPfehXzlK9N48sm1zJw5i23bfkV19SHWr/8F0HzBjxo1us3Xyc8vYODAf2HUqKs+\nvSfXAzif3OVsctMHwF+g9SXh/0XzS+cHApcBPwD+h9PvBfsm8BTNr1K8k+ZYbeP0KxOh+WX304C1\nwCzgV8Ah4BcX7ml8KnpExBYtepAPP2xg4cJvcOLE+0yZMpUXXthBXl5e65533z1Gnz59Wj+fNCnF\nM89sZsWKpTz66FJGjixhy5YX2118Z/InyI/H+eQuZ5ObDgLTaX6PWAL4Xsv6HcAzNL8Y49gZ+4cD\nvwMWAeuAS4FNtH01YgrYDCxt+SgBXgSyTy2GHvE+MUn6OCK8T6xX6w3vE5Mk9V5GTJIUlhGTJIVl\nxCRJYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVl\nxCRJYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVl\nxCRJYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVl\nxCRJYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhTWxd19AvrkLinIdPcp6BzqP0h09ykoiwRe\nO7msK9PxTkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJ\nYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJ\nYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJ\nYRkxSVJYRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUVo+J2GOPPUxJ\nyVCKivKZPfuLvP32Xzo9Zvv2rYwffxWDB/cnlSrl5Zd3ZN27Zs0qCgsv4qGHFp/P0+7xnnrqKa4Y\nMYL++flMTqV49dVXz7m/srKS8RMm8Jn+/fn8lVfy3HPPtduzdetWrho9mv75+ZSOHcuOHdnnps55\n7eSq7cBMYBDN36rf6OJxW4GrgP5AKdDRbJ4CrmjZMxk493WZy3pExNauXU15+XrWrSunsvIA+fkF\nzJ07k8bGxqzH7Nu3lzvvvI0FC+5mz55qZs2aw/z5N/PWW2+223vo0Ks8+2w5V19deiGfRo/z61//\nmu99//usWL6c1197jdIxY5j55S9z/PjxDvcfPXqUr950EzfMmMHh6moW3n8/d919Nzt37mzds3fv\nXm77+te5+667qH79debMns3Nc+fy5pvt56bOee3ksg+AqcBPgEQXj9kL3AbcDVQDc4CbgTNn82vg\ne8AK4HWaQzcT6Pi6zHWJTCbTpY319XRtYzcoKRnKwoUPcN99iwCoq6tj5MgkGzc+xy233NrhMQsW\nzKOhoYGKipda12bMSFFaOo7HH9/QulZfX8/UqeN54omfs3r1SsaMGceqVWsv7BP6J11SkJujmZxK\nMWniRH72s58BkMlkuOzyy7n/O9/hwQcfbLd/yZIl7PjDH3jj8OHWtfm33caJEyf4/e9+B8C8+fNp\naGjgpRdfbN2TmjKFcWPHsmHDhnaPmQvqP+jqN6BPX2+/dgYM6O4z6Ir/pvmuqRoY08neeUAD8NIZ\naylgHPDRbCYDk4CftXyeAS4D7gfaX5fdKZPpvN7h78SOHv0r6XQN06bd0LpWWFjIhAmTOHCgKutx\nBw5UMX16WZu1srKZ7Y5ZvPjb3HjjTVx//Yzze+I93KlTpzh06BA33HB6LolEgrKyMqr27evwmH37\n91N2xn6AmV/6ElVVp2dSVVXV8Z4sj6nsvHZ6oiqg7Ky1mS3rAKeAQ8CZ11Ci5ZjsM89lF3f3CXxS\n6XQNiUSCoqJkm/WioiTpdM05j+vsmG3btnDkSDW7dx88vyfdCxw/fpympiaSybb/xsmiIv785z93\neExNTU37/ckkdXV1nDx5kn79+mXdU1OTfdbqmNdOT1QDJM9aS7asQ/OvDJuy7On4usx14e7EKio2\nM2TIAIYMGcDQoYWcOnXqgnydd945xpIl32XTpufp27fvBfka0qfJayeXbQYGtHwUAnu693QCCXcn\nNmvWHK69dnLr5ydP/oNMJkNtbbrNT4e1tWlKS8dlfZxkspja2nSbtdraNMlkMQDV1a9x/PjfuO66\na/jo74ZNTU3s2bOb8vL1vPfeSRKJ3P1bR3cbNGgQffr0IZ1u+2+crq2luLi4w2OKi4vb70+nKSws\npF+/fufck+0xdZrXTi6bQ/Pfqj4y7GM+TjGQPmst3bIOza907NPJnljC3YkVFBRwxRUjWj9GjRpN\nMllMZeWu1j11dXUcPLifSZOmZH2ciRNTbY4BeOWVnUycmAJg+vQy9u8/wt691VRVHaaq6jDXXDOB\nr33tdvbuPexF2Im+ffsyfvx4du06/W+cyWTYtWsXU1KpDo9JTZ7MrldeabP28s6dpM7Yn0ql2u3Z\n+ac/kZo8GZ2b104uKwBGnPHR76z/3tV/sxSw66y1nS3rAH2B8WftybR8nn3muazP8uXLu7SxsZGu\nbewGTU1NrFnzY668cjSNjY088MD9NDae5Kc/XUefPn0AuOeeOzh06NXWP2IPGTKMlSuXkZ9fwGc/\nO5CNG9ezfftWNmzYxKBBg8nLy2PQoMFtPioqNjN8+Ajmzbu9O59uO3l53X0GHSssLOThRx7hsssu\no1+/fixbtozDb7zB008/TUFBAT/4wQ/491/+krk33wzA5z73OX70ox9x4v33ufzyy6moqGDt44+z\n/sknGTFiBADDhg1j2Q9/SEF+PgMHDmT9+vVs3bqVTZs2MXjw4O58ulk1nsrdb9q9/dr58Y+7+wzO\n5X+B/wTepvnXjdOA/6M5aJe07LmD5vd4ffRCjWHAMpqjOBBYT/P7xjYBH10fhcDDNL8isV/L/sPA\n0y3H5Y7ly1nR2Z5wv07syKJFD/Lhhw0sXPgNTpx4nylTpvLCCzvIO+O7+7vvHmu9KAEmTUrxzDOb\nWbFiKY8+upSRI0vYsuVFRo0anfXr+BPkP+fWW2/l+PHjPPzII6TTacaOHcsf//CH1tjUpNMcO3as\ndf/w4cP53W9/y6LFi1n35JNceumlbHr6acrKTr/aKpVKsfn551m6bBlLly2jpKSEF3/zG0aPzj43\nZee1k8teAv6N5mglgPkt64/QHCGAYzT/evAjKZqDt7TlowR4EThzNrfS/AKPh2n+NeJY4I+cjlws\nPeJ9Yr1drr5PTM1y+X1ivV2M94n1Xr3ifWKSpN7LiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKk\nsIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKk\nsIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKk\nsIyYJCksIyZJCsuISZLCMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKk\nsIyYJCksIyZJCsuISZLCMmKSpLAu7u4T0CdX/0Giu09BCunvf+/uM9An5Z2YJCksIyZJCsuISZLC\nMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLC\nMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLC\nMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKksIyYJCksIyZJCsuISZLC\nMmKSpLCMmCQpLCMmSQrLiEmSwjJikqSwekzEHnvsYUpKhlJUlM/s2V/k7bf/0ukx27dvZfz4qxg8\nuD+pVCkvv7wj6941a1ZRWHgRDz20+Hyedq/hfHKXs8ldzqZzPSJia9euprx8PevWlVNZeYD8/ALm\nzp1JY2Nj1mP27dvLnXfexoIFd7NnTzWzZs1h/vybeeutN9vtPXToVZ59tpyrry69kE+jx3I+ucvZ\n5C5n0zWJTCbTpY319XRtYzcoKRnKwoUPcN99iwCoq6tj5MgkGzc+xy233NrhMQsWzKOhoYGKipda\n12bMSFFaOo7HH9/QulZfX8/UqeN54omfs3r1SsaMGceqVWsv7BPqYZxP7nI2ucvZwCWXkOhsT/g7\nsaNH/0o6XcO0aTe0rhUWFjJhwiQOHKjKetyBA1VMn17WZq2sbGa7YxYv/jY33ngT118/4/yeeC/h\nfHKXs8ldzqbrLu7uE/ik0ukaEokERUXJNutFRUnS6ZpzHtfZMdu2beHIkWp27z54fk+6F3E+ucvZ\n5C5n03Xh7sQqKjYzZMgAhgwZwNChhZw6deqCfJ133jnGkiXfZdOm5+nbt+8F+Ro9kfPJXc4mdzmb\njy/cndisWXO49trJrZ+fPPkPMpkMtbXpNj+B1NamKS0dl/VxksliamvTbdZqa9Mkk8UAVFe/xvHj\nf+O6667ho78bNjU1sWfPbsrL1/PeeydJJDr9dW2v43xyl7PJXc7m4wsXsYKCAq64YkSbtWSymMrK\nXXzhC2OA5j+AHjy4n3vu+XbWx5k4MUVl5S7uvff+1rVXXtnJxIkpAKZPL2P//iNtjvnmNxfw+c9f\nxeLFD4Ub9KfF+eQuZ5O7nM3H12f58uVd2tjYSNc2doOmpibWrPkxV145msbGRh544H4aG0/y05+u\no0+fPgDcc88dHDr0ausfSocMGcbKlcvIzy/gs58dyMaN69m+fSsbNmxi0KDB5OXlMWjQ4DYfFRWb\nGT58BPPm3d6dTzcc55O7nE3ucjaQl8eKzvaEuxPryKJFD/Lhhw0sXPgNTpx4nylTpvLCCzvIy8tr\n3fPuu8daBw8waVKKZ57ZzIoVS3n00aWMHFnCli0vMmrU6KxfJ+JPKbnA+eQuZ5O7nE3X9Ij3iUmS\nep5e8T4xSVLvZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJYRkxSVJY\nRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJYRkxSVJY\nRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJYRkxSVJY\nRkySFJYRkySFZcQkSWEZMUlSWEZMkhSWEZMkhWXEJElhGTFJUlhGTJIUlhGTJIVlxCRJYRkxSVJY\niUwm093nIEnSx+KdmCQpLCMmSQrLiEmSwjJikqSwjJgkKSwjJkkKy4hJksIyYpKksIyYJCms/wfu\n3O6n9BPwdwAAAABJRU5ErkJggg==\n", "text/plain": [ - "" + "" ] }, "metadata": {}, "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Widget Javascript not detected. It may not be installed or enabled properly.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Widget Javascript not detected. It may not be installed or enabled properly.\n" + ] } ], "source": [ @@ -467,6 +483,102 @@ "source": [ "Move the slider above to observe how the utility changes across iterations. It is also possible to move the slider using arrow keys or to jump to the value by directly editing the number with a double click. The **Visualize Button** will automatically animate the slider for you. The **Extra Delay Box** allows you to set time delay in seconds upto one second for each time step." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#POLICY ITERATION\n", + "\n", + "The `policy_iteration(mdp)` function follows closely from fig. 17.7 pseudocode.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def policy_iteration(mdp):\n", + " \"\"\"Solve an MDP by policy iteration [Figure 17.7]\"\"\"\n", + " U = {s: 0 for s in mdp.states}\n", + " pi = {s: random.choice(mdp.actions(s)) for s in mdp.states}\n", + " while True:\n", + " U = policy_evaluation(pi, U, mdp)\n", + " unchanged = True\n", + " for s in mdp.states:\n", + " a = argmax(mdp.actions(s), key=lambda a: expected_utility(a, s, U, mdp))\n", + " if a != pi[s]:\n", + " pi[s] = a\n", + " unchanged = False\n", + " if unchanged:\n", + " return pi" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Given a policy `pi` , the `policy_evaluation` function calculates the utility of each state if `pi` were to be executed. The ith iteration specifies the action to be taken in state `s`. We can perform any no. of iterations (`k=` 50 here) to give a reasonably good approximation of utilities." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def policy_evaluation(pi, U, mdp, k=50):\n", + " \"\"\"Return an updated utility mapping U from each state in the MDP to its\n", + " utility, using an approximation (modified policy iteration).\"\"\"\n", + " R, T, gamma = mdp.R, mdp.T, mdp.gamma\n", + " for i in range(k):\n", + " for s in mdp.states:\n", + " U[s] = R(s) + gamma * sum([p * U[s1] for (p, s1) in T(s, pi[s])])\n", + " return U" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's run the algorithm :" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{(0, 0): (0, 1),\n (0, 1): (0, 1),\n (0, 2): (1, 0),\n (1, 0): (1, 0),\n (1, 2): (1, 0),\n (2, 0): (0, 1),\n (2, 1): (0, 1),\n (2, 2): (1, 0),\n (3, 0): (-1, 0),\n (3, 1): None,\n (3, 2): None}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "policy_iteration(sequential_decision_environment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] } ], "metadata": { @@ -478,7 +590,7 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 3.0 }, "file_extension": ".py", "mimetype": "text/x-python", @@ -510,7 +622,7 @@ "022a5fdfc8e44fb09b21c4bd5b67a0db": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -541,7 +653,7 @@ "0675230fb92f4539bc257b768fb4cd10": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -557,7 +669,7 @@ "0783e74a8c2b40cc9b0f5706271192f4": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -585,7 +697,7 @@ "098f12158d844cdf89b29a4cd568fda0": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -610,7 +722,7 @@ "0b65fb781274495ab498ad518bc274d4": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -719,7 +831,7 @@ "1af711fe8e4f43f084cef6c89eec40ae": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -735,7 +847,7 @@ "1c5c913acbde4e87a163abb2e24e6e38": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -760,7 +872,7 @@ "200e3ebead3d4858a47e2f6d345ca395": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -878,7 +990,7 @@ "2d3acd8872c342eab3484302cac2cb05": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -888,7 +1000,7 @@ "2e1351ad05384d058c90e594bc6143c1": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -901,7 +1013,7 @@ "2f5438f1b34046a597a467effd43df11": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -938,7 +1050,7 @@ "319425ba805346f5ba366c42e220f9c6": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -957,7 +1069,7 @@ "332a89c03bfb49c2bb291051d172b735": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1006,7 +1118,7 @@ "388571e8e0314dfab8e935b7578ba7f9": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1028,7 +1140,7 @@ "3a21291c8e7249e3b04417d31b0447cf": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1041,7 +1153,7 @@ "3b22d68709b046e09fe70f381a3944cd": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1051,7 +1163,7 @@ "3c1b2ec10a9041be8a3fad9da78ff9f6": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1076,7 +1188,7 @@ "3e5b9fd779574270bf58101002c152ce": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1086,7 +1198,7 @@ "3e8bb05434cb4a0291383144e4523840": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1135,7 +1247,7 @@ "428e42f04a1e4347a1f548379c68f91b": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1151,7 +1263,7 @@ "4379175239b34553bf45c8ef9443ac55": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1164,7 +1276,7 @@ "4421c121414d464bb3bf1b5f0e86c37b": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1195,7 +1307,7 @@ "4731208453424514b471f862804d9bb8": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1244,7 +1356,7 @@ "4d281cda33fa489d86228370e627a5b0": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1263,7 +1375,7 @@ "4ec035cba73647358d416615cf4096ee": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1288,7 +1400,7 @@ "5141ae07149b46909426208a30e2861e": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1325,7 +1437,7 @@ "55a1b0b794f44ac796bc75616f65a2a1": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1386,7 +1498,7 @@ "595c537ed2514006ac823b4090cf3b4b": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1447,7 +1559,7 @@ "5f823979d2ce4c34ba18b4ca674724e4": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1487,14 +1599,14 @@ "644dcff39d7c47b7b8b729d01f59bee5": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, "6455faf9dbc6477f8692528e6eb90c9a": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1507,7 +1619,7 @@ "665ed2b201144d78a5a1f57894c2267c": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1550,7 +1662,7 @@ "6a28f605a5d14589907dba7440ede2fc": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1575,7 +1687,7 @@ "6d7effd6bc4c40a4b17bf9e136c5814c": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1624,7 +1736,7 @@ "72dfe79a3e52429da1cf4382e78b2144": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1655,7 +1767,7 @@ "75e344508b0b45d1a9ae440549d95b1a": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1713,7 +1825,7 @@ "7f2f98bbffc0412dbb31c387407a9fed": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1744,7 +1856,7 @@ "82e2820c147a4dff85a01bcddbad8645": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1847,21 +1959,21 @@ "8cffde5bdb3d4f7597131b048a013929": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, "8db2abcad8bc44df812d6ccf2d2d713c": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, "8dd5216b361c44359ba1233ee93683a4": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1907,7 +2019,7 @@ "933904217b6045c1b654b7e5749203f5": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1935,7 +2047,7 @@ "94f2b877a79142839622a61a3a081c03": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1957,7 +2069,7 @@ "97207358fc65430aa196a7ed78b252f0": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -1970,7 +2082,7 @@ "986c6c4e92964759903d6eb7f153df8a": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2013,14 +2125,14 @@ "9d5e9658af264ad795f6a5f3d8c3c30f": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, "9d7aa65511b6482d9587609ad7898f54": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2039,7 +2151,7 @@ "9efb46d2bb0648f6b109189986f4f102": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2055,7 +2167,7 @@ "9f43f85a0fb9464e9b7a25a85f6dba9c": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2068,7 +2180,7 @@ "9faa50b44e1842e0acac301f93a129c4": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2093,7 +2205,7 @@ "a1840ca22d834df2b145151baf6d8241": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2130,7 +2242,7 @@ "a39cfb47679c4d2895cda12c6d9d2975": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2161,7 +2273,7 @@ "a87c651448f14ce4958d73c2f1e413e1": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2270,7 +2382,7 @@ "b7e4c497ff5c4173961ffdc3bd3821a9": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2295,7 +2407,7 @@ "b9c138598fce460692cc12650375ee52": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2314,7 +2426,7 @@ "bbe5dea9d57d466ba4e964fce9af13cf": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2348,7 +2460,7 @@ "beb0c9b29d8d4d69b3147af666fa298b": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2415,7 +2527,7 @@ "c74bbd55a8644defa3fcef473002a626": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2482,7 +2594,7 @@ "ce3a0e82e80d48b9b2658e0c52196644": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2492,7 +2604,7 @@ "ce8d3cd3535b459c823da2f49f3cc526": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2562,7 +2674,7 @@ "d83329fe36014f85bb5d0247d3ae4472": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2596,7 +2708,7 @@ "dc7376a2272e44179f237e5a1c7f6a49": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2693,7 +2805,7 @@ "e4e5dd3dc28d4aa3ab8f8f7c4a475115": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2709,7 +2821,7 @@ "e64ab85e80184b70b69d01a9c6851943": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2806,7 +2918,7 @@ "f262055f3f1b48029f9e2089f752b0b8": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2837,7 +2949,7 @@ "f3df35ce53e0466e81a48234b36a1430": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2916,7 +3028,7 @@ "f9458080ed534d25856c67ce8f93d5a1": { "views": [ { - "cell_index": 27 + "cell_index": 27.0 } ] }, @@ -2977,4 +3089,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file