From c6e0ee5c05bcb09680daf98dc669052f1670412e Mon Sep 17 00:00:00 2001 From: Arthur DANJOU Date: Fri, 30 Jan 2026 14:49:47 +0100 Subject: [PATCH] Implement feature X to enhance user experience and fix bug Y in module Z --- ...and Figure 8_2 in [Sutton and Barto].ipynb | 522 ++++++++++++++++++ 1 file changed, 522 insertions(+) create mode 100644 M2/Reinforcement Learning/Lab 6A Dyna-Q and Figure 8_2 in [Sutton and Barto].ipynb diff --git a/M2/Reinforcement Learning/Lab 6A Dyna-Q and Figure 8_2 in [Sutton and Barto].ipynb b/M2/Reinforcement Learning/Lab 6A Dyna-Q and Figure 8_2 in [Sutton and Barto].ipynb new file mode 100644 index 0000000..53cea94 --- /dev/null +++ b/M2/Reinforcement Learning/Lab 6A Dyna-Q and Figure 8_2 in [Sutton and Barto].ipynb @@ -0,0 +1,522 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27bda6b2", + "metadata": {}, + "source": [ + "\n", + "# Lab6A: Dyna-Q and Figure 8.2 in [Sutton & Barto]\n", + "\n", + "This lab reproduces **Figure 8.2** from *Reinforcement Learning: An Introduction*\n", + "(Sutton & Barto, Chapter 8).\n", + "\n", + "We study **Dyna-Q** in a simple deterministic maze and analyze how the number of\n", + "planning steps `n` affects learning speed, measured in **steps per episode**.\n", + "\n", + "You will:\n", + "- Build the **Dyna Maze** (47 states)\n", + "- Implement a **Dyna-Q algorithm**\n", + "- Reproduce the learning curves for `n = 0, 5, 50`\n", + "- Understand *why* planning accelerates learning\n", + "\n", + "---\n" + ] + }, + { + "cell_type": "markdown", + "id": "009b4a82", + "metadata": {}, + "source": [ + "\n", + "## 1. Experimental setup\n", + "\n", + "- Discount factor: $\\gamma = 0.95$\n", + "- Learning rate: $\\alpha = 0.1$\n", + "- Exploration: $\\varepsilon = 0.1$ for $\\varepsilon$-greedy.\n", + "- Reward:\n", + " - `+1` when entering the goal state\n", + " - `0` otherwise\n", + "- Environment:\n", + " - Deterministic transitions\n", + " - Grid world with obstacles (47 states total)\n", + "\n", + "These choices match the RL book setup for Figure 8.2 in [Sutton & Barto].\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ef6587ce", + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from typing import ClassVar\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n" + ] + }, + { + "cell_type": "markdown", + "id": "c2b72d28", + "metadata": {}, + "source": [ + "**Exercise 1.** (The Dyna Maze Environment) Implement the maze used in Figure 8.2 and understand the code.\n", + "\n", + "The maze:\n", + "- Is a 6×9 grid (54 cells)\n", + "- Contains **7 obstacles**, leaving **47 valid states**\n", + "- Has four actions: up, right, down, left\n", + "- If an action would hit a wall or obstacle, the agent **stays in place**\n", + "- Reward is +1 **only when entering the goal cell**\n" + ] + }, + { + "cell_type": "markdown", + "id": "1fe98da4", + "metadata": {}, + "source": [ + "\n", + "*Hints.*\n", + "\n", + "- Use a `(row, col) -> state_id` mapping\n", + "- Obstacles can be stored as a `set` of coordinates\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "58da6cb2-caf4-4759-85a7-1aaea0c051ee", + "metadata": {}, + "outputs": [], + "source": [ + "@dataclass # is the class header and automatically generates an __init__ for you using the annotated fields (rows, cols, gamma).\n", + "class DynaMazeEnv:\n", + " \"\"\"A simple gridworld maze environment for Dyna-Q experiments, matching Figure 8.2 in Sutton and Barto.\n", + "\n", + " Args:\n", + " rows (int): number of rows in the maze grid\n", + " cols (int): number of columns in the maze grid\n", + " gamma (float): discount factor for future rewards\n", + "\n", + " \"\"\"\n", + "\n", + " rows: int = (\n", + " 6 # Default maze size is 6x9, matching the maze used in Figure 8_2 example\n", + " )\n", + " cols: int = 9\n", + " gamma: float = 0.95 # discount factor\n", + "\n", + " action_moves: ClassVar[dict[int, tuple[int, int]]] = {\n", + " 0: (-1, 0), # up\n", + " 1: (0, 1), # right\n", + " 2: (1, 0), # down\n", + " 3: (0, -1), # left\n", + " }\n", + "\n", + " def __post_init__(self) -> None:\n", + " \"\"\"__post_init__ is called automatically after the dataclass __init__ method. It initializes the maze structure, walls, start/goal positions, and state mappings.\"\"\"\n", + " # 7 obstacles -> 54 - 7 = 47 states since\n", + " # Total grid cells: 6 * 9 = 54\n", + " # Wall cells: 7\n", + " # Valid states: 54 - 7 = 47\n", + "\n", + " self.walls = set() # self.walls stores blocked cells (walls) as (r, c) pairs.\n", + " for r in [\n", + " 1,\n", + " 2,\n", + " 3,\n", + " ]: # This part creates the maze's “two vertical walls + one extra block” structure.\n", + " self.walls.add((r, 2))\n", + " for r in [0, 1, 2]:\n", + " self.walls.add((r, 7))\n", + " self.walls.add((4, 5))\n", + "\n", + " self.start_rc = (2, 0) # Start is at row 2, col 0.\n", + " self.goal_rc = (0, 8) # Goal is at row 0, col 8.\n", + "\n", + " # Now we map (row,col) to state index\n", + " self.rc_to_s = {}\n", + " self.s_to_rc = {}\n", + " s = 0\n", + " for r in range(self.rows): # Iterate through all cells\n", + " for c in range(self.cols):\n", + " if (\n", + " (r, c) in self.walls\n", + " ): # If the cell is a wall, skip it: walls are not states.\n", + " continue\n", + " self.rc_to_s[(r, c)] = (\n", + " s # Otherwise, rc_to_s[(r,c)] = s gives an integer label for this free cell.\n", + " )\n", + " self.s_to_rc[s] = (\n", + " r,\n", + " c,\n", + " ) # s_to_rc[s] = (r,c) lets you recover the grid coordinate from a state id.\n", + " s += 1 # s increments only for valid cells.\n", + "\n", + " assert s == 47, ( # noqa: PLR2004, S101\n", + " f\"Expected 47 states, got {s}\"\n", + " ) # The assert prevents silent bugs if walls are miscounted.\n", + " self.n_states = s # number of states\n", + " self.n_actions = 4 # number of actions\n", + " self.reset() # self.reset() sets the initial state.\n", + "\n", + " def reset(self) -> int:\n", + " \"\"\"Reset the environment to the start state.\"\"\"\n", + " self.state = self.rc_to_s[self.start_rc]\n", + " return self.state\n", + "\n", + " def step(self, action: int) -> tuple[int, float, bool]:\n", + " \"\"\"Take an action in the environment. Returns (next_state, reward, done).\"\"\"\n", + " r, c = self.s_to_rc[\n", + " self.state\n", + " ] # self.state is an integer, so we need to convert it to grid coordinate.\n", + " dr, dc = self.action_moves[action]\n", + " nr, nc = r + dr, c + dc # Compute proposed next cell (nr, nc).\n", + "\n", + " if ( # Boundary + wall collision handling : If the agent try to go outside the grid or into a wall, the agent stays in place.\n", + " nr < 0\n", + " or nr >= self.rows\n", + " or nc < 0\n", + " or nc >= self.cols\n", + " or (nr, nc) in self.walls\n", + " ):\n", + " nr, nc = r, c # The agent should stay in the same place.\n", + "\n", + " next_state = self.rc_to_s[(nr, nc)] # convert back to integer state\n", + " reward = 1.0 if (nr, nc) == self.goal_rc else 0.0 # Reward: Goal = 1, other = 0\n", + " done = (\n", + " nr,\n", + " nc,\n", + " ) == self.goal_rc # termination : Episode ends when goal is reached.\n", + " self.state = next_state # Update self.state\n", + " return next_state, reward, done # return transition tuple\n" + ] + }, + { + "cell_type": "markdown", + "id": "2504416c", + "metadata": {}, + "source": [ + "\n", + "**Exercise 2.** (Implement Dyna-Q) Implement a **tabular Dyna-Q algorithm**:\n", + "\n", + "Algorithm structure:\n", + "1. Select action using $\\varepsilon$-greedy with fair tie-breaking.\n", + "2. Take real step in environment\n", + "3. Q-learning update\n", + "4. Store transition in the model\n", + "5. Perform `n` planning updates by sampling from the model\n" + ] + }, + { + "cell_type": "markdown", + "id": "adc560ec", + "metadata": {}, + "source": [ + "\n", + "### Hints\n", + "\n", + "- Initialize all Q-values to zero\n", + "- Use a dictionary for the model: `(s, a) -> (r, s')`\n", + "- Planning updates are **identical** to Q-learning updates\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1a5f489d", + "metadata": {}, + "outputs": [], + "source": [ + "def epsilon_greedy_action(\n", + " Q_row: np.ndarray,\n", + " eps: float,\n", + " rng: np.random.Generator,\n", + ") -> int:\n", + " \"\"\"Implement an ε-greedy policy. Here, Q_row is the vector Q(s, .) for a fixed state s.\"\"\"\n", + " if rng.random() < eps: # With probability epsilon: explore (choose a random action)\n", + " return int(\n", + " rng.integers(0, len(Q_row)),\n", + " ) # With probability 1-ε: exploit (choose an action with the highest estimated value)\n", + " maxv = np.max(Q_row) # Finds the maximum Q-value among actions.\n", + " best = np.flatnonzero(\n", + " Q_row == maxv,\n", + " ) # here can be ties, means several actions have the same Q value,\n", + " # Q_row == maxv gives a boolean array like [True, False, True, ...]\n", + " # np.flatnonzero(...) returns the indices where it's True (a 1D array), e.g. [0, 2]\n", + " return int(\n", + " rng.choice(best),\n", + " ) # If multiple best actions exist, pick one uniformly at random and this prevents a systematic bias like always choosing the smallest index action.\n", + "\n", + "\n", + "class DynaQAgent:\n", + " \"\"\"Now we define a learning agent that implements the Dyna-Q algorithm.\"\"\"\n", + "\n", + " def __init__( # noqa: PLR0913\n", + " self,\n", + " n_states: int,\n", + " n_actions: int,\n", + " alpha: float = 0.1,\n", + " gamma: float = 0.95,\n", + " eps: float = 0.1,\n", + " planning_steps: int = 5,\n", + " rng: np.random.Generator | None = None,\n", + " ) -> None:\n", + " \"\"\"Initialize the Dyna-Q agent with parameters.\n", + "\n", + " Args:\n", + " n_states (int): number of states in the environment\n", + " n_actions (int): number of actions in the environment\n", + " alpha (float): learning rate for Q-learning updates\n", + " gamma (float): discount factor for future rewards\n", + " eps (float): epsilon for epsilon-greedy action selection\n", + " planning_steps (int): number of planning updates per real step\n", + " rng (np.random.Generator, optional): random number generator for reproducibility\n", + "\n", + " \"\"\"\n", + " self.Q = np.zeros(\n", + " (n_states, n_actions),\n", + " ) # This is the aabular Q-learning table with rows = states and columns = actions,\n", + " # Initially all values are 0\n", + " self.alpha = alpha # alpha = learning rate (step size) for Q updates.\n", + " self.gamma = gamma # gamma = discount factor used in TD target.\n", + " self.eps = eps # eps = epsilon for epsilon-greedy exploration.\n", + " self.n = planning_steps # This is the “n” in Dyna-Q: number of planning updates per real step.\n", + " self.rng = (\n", + " rng or np.random.default_rng()\n", + " ) # If we provide a random generator, use it (for reproducibility), otherwise, creat a new one.\n", + "\n", + " self.model = {} # This is the learned model is stored as a dictionary, with Key: (s, a) and value: (r, s'). Here we assume that the maze dynamics are deterministic.\n", + " self.observed_sa = [] # A list of all (s,a) pairs that have been seen at least once.\n", + " # Used so planning can sample only from experienced pairs (instead of impossible ones).\n", + "\n", + " def act(self, s: int) -> int:\n", + " \"\"\"Select an action using an ε-greedy policy based on the current Q-values.\"\"\"\n", + " return epsilon_greedy_action(\n", + " self.Q[s],\n", + " self.eps,\n", + " self.rng,\n", + " ) # self.Q[s] extracts the row vector Q(s, :) (the value of actions for state s) then we pass it to epsilon-greedy.\n", + "\n", + " def q_update(self, s: int, a: int, r: float, sp: int) -> None:\n", + " \"\"\"Perform a Q-learning update for a given transition.\"\"\"\n", + " td_target = r + self.gamma * np.max(self.Q[sp]) # sp here means s'\n", + " self.Q[s, a] += self.alpha * (td_target - self.Q[s, a]) # Q-learning update\n", + "\n", + " def observe(\n", + " self,\n", + " s: int,\n", + " a: int,\n", + " r: float,\n", + " sp: int,\n", + " ) -> None:\n", + " \"\"\"Observe a transition and update the agent's knowledge.\"\"\"\n", + " # Direct RL update\n", + " self.q_update(\n", + " s,\n", + " a,\n", + " r,\n", + " sp,\n", + " ) # Direct RL update : this is learning from real experience (interaction with the environment).\n", + "\n", + " # Model update = this is learning from simulated experience\n", + " key = (s, a) # key identifies the state-action pair.\n", + " if key not in self.model: # If this (s,a) has never been seen,\n", + " self.observed_sa.append(\n", + " key,\n", + " ) # record it in observed_sa so planning can use it.\n", + " self.model[key] = (\n", + " r,\n", + " sp,\n", + " ) # Now we update the model : “If in state s take action a, we will get reward r and go to state sp.”\n", + "\n", + " # Planning = This is the “Dyna” part.\n", + " for _ in range(self.n): # for each planning step (n steps in total)\n", + " ss, aa = self.observed_sa[\n", + " int(self.rng.integers(len(self.observed_sa)))\n", + " ] # Sample a previously observed (s,a) pair uniformly at random\n", + " rr, ssp = self.model[\n", + " (ss, aa)\n", + " ] # Use the model to retrieve its predicted reward and next state (rr, ssp). Hint. use self.model[key], what is the key here ?\n", + " self.q_update(\n", + " ss,\n", + " aa,\n", + " rr,\n", + " ssp,\n", + " ) # Apply the same Q-learning update as if it were a real transition. Hint. use self.q_update(ss, aa, rr, ssp)" + ] + }, + { + "cell_type": "markdown", + "id": "2688c480", + "metadata": {}, + "source": [ + "\n", + "**Exercise 3.** Reproduce Figure 8.2\n", + "\n", + "Run the experiment:\n", + "- 50 episodes\n", + "- Average over 30 independent runs\n", + "- Compare `n = 0`, `n = 5`, `n = 50`\n", + "- Plot **steps per episode vs episodes**\n" + ] + }, + { + "cell_type": "markdown", + "id": "a6589733", + "metadata": {}, + "source": [ + "\n", + "*Hints*\n", + "\n", + "- Use the **same random seed across different `n` values** for each repetition\n", + "- Episode 1 should be identical for all `n`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "206bf2bb", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAW35JREFUeJzt3Qd4VGXWB/D/9PQEEgi9iiIgoDQB2wqCXRRXV1FxZWWtH8ruurqroKKiqNgXRCy4q4LYVlFZERFQOogiIhaqQOjpydT7Pee9cycTkkAmTGPm/3u83jslM5ebdnLe877HpGmaBiIiIqIEZY71CRARERFFEoMdIiIiSmgMdoiIiCihMdghIiKihMZgh4iIiBIagx0iIiJKaAx2iIiIKKFZY30C8cDn82Hnzp3IzMyEyWSK9ekQERFRPchSgSUlJWjRogXM5rrzNwx2ABXotG7dOtanQURERA2wfft2tGrVqs7HGewAKqNjXKysrKxYnw4RERHVQ3FxsUpWGL/H68JgBwgMXUmgw2CHiIjo2HKkEhQWKBMREVFCY7BDRERECY3BDhERESU01uwQERGFidfrhdvtjvVpJAybzQaLxXLUr8Ngh4iIKAzrvRQUFKCwsDDWp5JwcnJy0KxZs6NaB4/BDhER0VEyAp2mTZsiLS2NC9SGKYAsLy/Hnj171O3mzZs3+LUY7BARER3l0JUR6OTm5sb6dBJKamqq2kvAI9e3oUNaLFAmIiI6CkaNjmR0KPyM63o0tVAMdoiIiMKAQ1fxe10Z7BAREVFCY7BDRERECY3BDhERESU0BjsRVFLpxi97SlDp9sb6VIiIiCLuu+++w+mnn46UlBTVjXzSpEmIBwx2Imjw5IUYPHkRftpdEutTISIiiqji4mIMGTIEbdu2xerVq/H444/j/vvvx7Rp0xBrXGcngpplpWB3sRMFRZXo3irWZ0NERNFaDK8iRhn9VJslpNlLZ511Frp3764yMdOnT4fdbsdNN92kgpRQvfHGG3C5XHjllVfU63Tt2hVr167F5MmTMXr0aMQSg50Iys9KAVCE3cWVsT4VIiKKEgl0uoz7X0ze+4cHhyLNHtqv9hkzZmDs2LFYvnw5li5diuuvvx4DBw7EOeecg/POOw+LFy+u82Mli7N+/Xp1LB97xhlnqEDHMHToUDz22GM4ePAgGjVqhFhhsBNBzbIl2AEKGOwQEVGc6t69O8aPH6+OO3XqhOeffx7z589XwY5keyoqKg7bqDO4ZUb79u2rPZ6fnx94jMFOQmd2gIIiZ6xPhYiIojiUJBmWWL13Q4KdYNKDyuhH1bJlSyQCBjsRrtkRHMYiIkoeUjMT6lBSLNmCsjPG+ft8PnUcyjCWdCbfvXt3tceN2/JYLB07n41jEIexiIjoWDY9hGGs/v3745///KfqYWXcP2/ePJxwwgkxHcISDHaiMIy1u4jBDhERHXtahjCMdfXVV+OBBx7AqFGj8Pe//x3ff/89nnnmGTz11FOINQY7UcjslDg9KHN6kO7g5SYiosSUnZ2Nzz77DLfeeit69eqFvLw8jBs3LubTzgV/+0ZQhsOqtlKnRw1ldWySEetTIiIiCvjyyy9xqA8++AANJcXOh6vxiRWuoBxJ+3/FwPQdSEUlh7KIiIhihMFOJP1nOF4svxMnmraxSJmIiChGGOxEklWv2UkxuRjsEBERxQhrdiJoXKoHO5s1ha9gH4exiIiIYoSZnQj6xuzF8tQUmC1lzOwQERHFCIOdCHKY9MtrUcNYbBlBREQUCwx2Ishu0nuUWM1ODmMRERHFCIOdKAQ7ktnZW+qE16fF+pSIiIiSDoOdCLKb9Ppvi9mtAp19pRzKIiIiijYGOxHkMOvBTorNq/YFHMoiIqIEtWXLFtUx/dBt2bJlsT41Tj2PJJtZ7/rqMIKd4kr0iPE5ERERRdLnn3+Orl27Bm7n5uYi1hjsRJDDH+zYrXqws5vTz4mIKI6cddZZqp9VSkoKpk+fDrvdjptuugn3339/g19TgptmzZohnnAYK4LsFj3YsZo5jEVElDQ0DXCVxWaT9w7RjBkzkJ6ejuXLl2PSpEl48MEHMW/ePPXYeeedh4yMjDq34AyO4eKLL0bTpk1x2mmn4cMPP0Q8YGYnguwWh35gqRrGIiKiBOcuBx5pEZv3/sdOwJ4e0od0794d48ePV8edOnXC888/j/nz5+Occ85R2Z6Kioo6P9Zm0/+oFxL8PPnkkxg4cCDMZjPeffddDBs2THVRlwAolhjsRJDdbFd7k4nDWEREFJ+6d+9e7Xbz5s2xZ88eddyyZct6v05eXh7Gjh0buN2nTx/s3LkTjz/+OIOdROaw6pkdzeRRew5jERElAVuanmGJ1XuH+iG2quyMkBlUPp8vMIy1ePHiOj+2bdu2WL9+fZ2P9+vXLzAkFksMdiLIZtG7nvtgZHa4zg4RUcIzmUIeSopX00MYxqrN2rVrVaYo1hjsRJDDqgc7Hv8wVqnTo7YMBy87ERHFv5YhDGNJobPM5jr55JPV7ffeew+vvPKKCphijb91I8huBDuaF5kOK0qcHjWUdVzTjFifGhERUdhNmDABW7duhdVqRefOnTFr1ixcfvnliDUGOxFkt6aqvVPzIj87BSV7SlWRMoMdIiKKB19++WWN+2T2VEOMHDlSbfGI6+xEkMNfKObSvGiWpWd5WKRMREQUXQx2IshuNYIdH/KNYIfTz4mIiKKKwU4E2f3V+E740Cxbn4bOtXaIiIiii8FOBNmNYSxoHMYiIiKKEQY7EeSwZQSCnfxMZnaIiIhigcFOFIaxXCYTmmeY1DFrdoiIiJIo2PF6vbjvvvvQvn17pKamomPHjmqOvhbUtVWOx40bp1ZglOcMHjwYP//8c7XXOXDgAEaMGIGsrCzk5ORg1KhRKC0tRazZ7Zlq7zSZ0My/gvfeEic8Xn0ZbiIiIkrwYOexxx7DlClTVIfVDRs2qNvSXv65554LPEduP/vss5g6dapqPy9t6IcOHYrKyqoMiQQ60ptD+m/MmTMHixYtwujRoxEvNTtukwmNHT5YzCb4NGBfqSvWp0ZERJQ0Yrqo4JIlS3DJJZfgggsuULfbtWuHt956CytWrAhkdZ5++mnce++96nni9ddfR35+vlr06A9/+IMKkubOnYuVK1eid+/e6jkSLJ1//vl44okn0KJFixrv63Q61WYoLi6OyL/PYXEEMjsWrxNNMx3YVVSphrKaZesFy0RERJTAmZ0BAwZg/vz5+Omnn9Ttb7/9Fl999ZXqsio2b96MgoICNXRlyM7OVl1Uly5dqm7LXoaujEBHyPPNZrPKBNVm4sSJ6nWMrXXr1hH599kstkDNDjyVVWvtcEYWERFRcgQ7d999t8rOSP8M6ZwqzcPuuOMONSwlJNARkskJJreNx2TftGnTao9LT47GjRsHnnOoe+65B0VFRYFt+/btEc3seEwm+FxlgennnJFFRESJZsuWLTCZTDW2ZcuWVXve7Nmz1e/9lJQUnHTSSfjkk08Sexjr7bffxhtvvIE333wTXbt2Va3gJdiRoadI9tdwOBxqizS7xR44dkmwk52njjkji4iIEtXnn3+ufqcbcnNzq5WvXHXVVWqE5cILL1S//4cNG4Y1a9agW7duiRns/O1vfwtkd4REeNItVS6CBDvNmjVT9+/evVvNxjLI7Z49e6pjec6ePXuqva7H41EztIyPj5XgYMfpLkF+Vit1vJvDWEREFAfOOussdO/eXWVZpk+fDrvdjptuugn3339/g19Tgpu6fv8+88wzOPfcc9XvfyEzsGVykUxUkolICTmMVV5ermprglksFvh8+tRsmZIuF0zqeoKLiaUWp3///uq27AsLC7F69erAc7744gv1GlLbE0tWkxUm/yx6t1MyO3o2iZkdIqLEJZNryt3lMdmCl26prxkzZqiZzvK7VWZAP/jggyoAEVJDm5GRUecWnMExXHzxxaq85LTTTsOHH35Y7TGpsw2uwxUyw9qow03IzM5FF12Ehx9+GG3atFEX7JtvvsHkyZNxww03qMdlrE+GtR566CF06tRJBT+yLo8Mc0naS5x44okqSrzxxhtVVOh2u3HbbbepbFFtM7GiSc7fARMqocHpLkV+DpuBEhElugpPBfq9GZs/tpdfvRxp/mVP6qt79+4YP368OpbftZJlkSTDOeeco7I9FRUVdX6s1NsaJPh58sknMXDgQJXIePfdd9Xvapk9LQGQkFraw9XhJmSwI1PEJXi55ZZb1FCUBCd//vOf1SKChrvuugtlZWVq3RzJ4EikKFPNJeVmkLofCXAGDRqkLvDw4cPV2jzxwGbSgx2Xu7yqQJnDWEREFCe6d+9e7baUjRjlIS1btqz36+Tl5WHs2LGB23369MHOnTvx+OOPB4KdWIlpsJOZmanW0ZHtcNkRSanJVheZeSVFTvHIATNK4IPLU47W/rV1ylxelFS6kZlSFRETEVFiSLWmqgxLrN47VLag7Izxe9coJ5FhrMWLF9f5sW3btlWL+tZFykmMITEhpSlSdxtMbke6xjamwU4ysJvMgCazscqRZrciM8WKkkqPmn7OYIeIKPFIsBDqUFK8mh7CMFZtZJZ18AQjqbOVITIpUTFIMGTU4UYKg50Is5ssKthxesrVbRnKKqksRUGRE8c11XtnERERxaOWIQxjSaGzzOaSNfPEe++9h1deeUUFTIYxY8bgzDPPVLU90j1h5syZWLVqFaZNm4ZIYrATjWBH1tnx6HU60ibi5z2lLFImIqKEM2HCBLWEjCzuKwsHzpo1C5dffnm1zglSdiJtoP7xj3+ogmgpYI7kGjuCwU6EOcxWwCvBjp4GNFpGcBVlIiKKtS+//LLGfRJ8NISsj1efBYF///vfqy2aYrrOTjKwmfR40uXVG48aM7LYH4uIiCg6GOxEmMOsF285/cNY+f4ZWRzGIiIiig4GOxFm93c+d3tdas9moERERNHFYCfC7Ga9P5aTw1hEREQxwWAnwhwWvR+Wy6dndvL9/bH2lTrh8eqLNhER0bGvIX2pKDrXlcFOlDqfu3xutc9Ld8BqNsGnAXtL9WwPEREdu4yF9aS5NYWfcV2PtIDh4XDqeYTZ/Zkdp1cPdsxmE5pmOrCzqFINZTXPDn1pbyIiih8WiwU5OTmBflJpaWlqFWUKQ/f48nJ1XeX6ynVuKAY7EWa3plTL7BgzsiTYYZEyEVFiMHo7GQEPhY8EOkfbO4vBToQ5jGBH8wbuY5EyEVFikUyO9IBq2rQp3O6qP27p6MjQ1dFkdAwMdiLMZtGHqVy+qmDHWEW5oJg1O0REiUR+MYfjlzOFFwuUI8xh0wMbp+YJ3Cf9sQSHsYiIiCKPwU6E2a1pau/WqqaZcxiLiIgoehjsRJjdpgc7TlQFO2wGSkREFD0MdqIU7LigAf66nWZB/bG4CBUREVFkMdiJMIc9Q+1dsuaCvxmoMYxV7vKixFlVy0NEREThx2Anwuy29Kpgx60HO6l2C7JS9IlweziURUREFFEMdqK0qKAzKLNTbSiriNPPiYiIIonBTpR6Y7ll5fCgYKdqrR1mdoiIiCKJwU6Uup7XyOxwRhYREVFUMNiJMJvFVqNmp/owFoMdIiKiSGKwE2EOsyNoNlZF4H4OYxEREUUHg50o1ewETz0XHMYiIiKKDgY7UQp2VM0Oh7GIiIiijsFOlIIdj8kEn7vmMNa+Uic83qpWEkRERBReDHaiNBtLuNylgePcdDtsFhN8GrC3lGvtEBERRQqDnQizm/XMjnC5ygLHZrMJTTM5lEVERBRpDHYizGq2QtYTFC53ebXH8rP0rA+LlImIiCKHwU6EmUwm2P2X2eWuyuwIFikTERFFHoOdKLCb9MvsrJHZMdbaYc0OERFRpDDYiQKHyaL2rqBFBQXX2iEiIoo8BjtRYDdZ1d4VtKig4DAWERFR5DHYiQK7Wc/sOL0VtQ5jMbNDREQUOQx2osBu8jcD9ThrHcaS/liapsXk3IiIiBIdg50ocJittQc7/mGscpcXJU5PTM6NiIgo0THYiQKb0QzUVz3YSbFZkJ2qZ312s26HiIgoIhjsRIHDv4qy0+Oq8VjwUBYRERGFH4OdKDYDdftqBjv5nJFFREQUUQx2osDubwbq9LlrPNYkQ3+MzUCJiIgig8FOFIMdl7dmZicnTa/ZKaqoGQgRERHR0WOwEwUOqz/Y0WrOuDIKlIsZ7BAREUUEg50osFn1uhyXr+5gh5kdIiKiyGCwEwUOa6raOzVvjcc4jEVERBRZDHaiwO4Pdty1BDtZ/sxOYTmDHSIiokhgsBMFdlua2juhAd7qQQ2HsYiIiCKLwU4U2G16ZsdlMgGHdD5nsENERBRZDHaiwGFLrwp23NWDnRx/sFNS6YHXx2agRERE4cZgJ5rr7KjMTkWtNTuC08+JiIjCj8FOFNtFOFWwU32lZJvFjHS7RR1zKIuIiCj8GOxEgd3fCNSthrGqZ3ZETpr+eCGDHSIiorBjsBMFDqM3Vi2ZneChLGZ2iIiIwo/BThTYLHow4zKhRs2OyE61qj2DHSIiovBjsBPFzE5ts7EEp58TERFFDoOdKBYo17bOjshJ1R8vKq/ZFZ2IiIiODoOdqM/GqiWzw/5YREREEcNgJwoc5uB1djiMRUREFE0MdqI9jHWYmh02AyUiIgo/BjtRH8aqbTYWMztERERxF+y4XC5s3LgRHo8nvGeUwMGOx2SCj7OxiIiI4jvYKS8vx6hRo5CWloauXbti27Zt6v7bb78djz76aCTOMWGmnguXu6zG4zksUCYiIoqfYOeee+7Bt99+iy+//BIpKSmB+wcPHoxZs2aF+/wSql2EcLnLazzOzA4REVHk6Ev3huCDDz5QQc2pp54Kk9Sg+EmW59dffw33+SUEq9kKuVKaBDueuoOdcpcXbq9PNQclIiKi8Aj5t+revXvRtGnTGveXlZVVC37qa8eOHbjmmmuQm5uL1NRUnHTSSVi1alXgcU3TMG7cODRv3lw9Lhmkn3/+udprHDhwACNGjEBWVhZycnLUMFtpaSnihVwXu0nvbO6qpRFoZooe7Ahmd4iIiGIc7PTu3Rsff/xx4LYR4EyfPh39+/cP6bUOHjyIgQMHwmaz4dNPP8UPP/yAJ598Eo0aNQo8Z9KkSXj22WcxdepULF++HOnp6Rg6dCgqK6sKfSXQWb9+PebNm4c5c+Zg0aJFGD16NOKJ3aQn0Zy1rLNjMZuQlaI/zunnREREMR7GeuSRR3DeeeepwERmYj3zzDPqeMmSJVi4cGFIr/XYY4+hdevWePXVVwP3tW/fvlpW5+mnn8a9996LSy65RN33+uuvIz8/Xw2n/eEPf8CGDRswd+5crFy5UgVi4rnnnsP555+PJ554Ai1atKjxvk6nU22G4uJiRJrdbAF8MoxVM9gxVlEurvQws0NERBTrzM5pp52GtWvXqkBHhpw+++wzNay1dOlS9OrVK6TX+vDDD1WA8vvf/169xsknn4yXXnop8PjmzZtRUFCghq4M2dnZ6Nevn3o/IXsZujICHSHPN5vNKhNUm4kTJ6rXMTYJuCLNYfJ3Pq9lnZ3gup1iBjtERESxzeyIjh07VgtKGmrTpk2YMmUKxo4di3/84x8qO/N///d/sNvtGDlypAp0hGRygslt4zHZH1pDZLVa0bhx48BzaptRJu8ZnNmJdMBjN+uX2uWtvdmn0Qy0sILNQImIiKIe7IQyzCNFwvXl8/lURkaGxoRkdr7//ntVnyPBTqQ4HA61xWQVZW8dw1jG9HPW7BAREUU/2JFhovrOtPJ6vfV+c5lh1aVLl2r3nXjiiXj33XfVcbNmzdR+9+7d6rkGud2zZ8/Ac/bs2VPtNWSITWZoGR8fD+xmPZhx+2oPZrICa+1wRWoiIqKoBzsLFiwIHG/ZsgV33303rr/++sDsK6mbmTFjhqqFCYXMxJKWE8F++ukntG3bNlCsLAHL/PnzA8GNZJmkFufmm29Wt+UcCgsLsXr16kDN0BdffKGyRlLbEy/s/lWUnXUMYwWagXIYi4iIKPrBzplnnhk4fvDBBzF58mRcddVVgfsuvvhiVaw8bdq0kIaf7rzzTgwYMEANY11xxRVYsWKFeg3ZhGST7rjjDjz00EPo1KmTCn7uu+8+NcNq2LBhgUzQueeeixtvvFENf7ndbtx2221qplZtM7Fi3vncV0fNDltGEBERxcdsLMniBM98Msh9EqyEok+fPnj//ffx1ltvoVu3bpgwYYKaai7r5hjuuusu1XdL1s2R58tigTLVPLhVxRtvvIHOnTtj0KBBasq5zBgzAqZ44bDq5+vy1T5MxdlYREREcTIbS2YtyUwsWewvmCwq2JAZTRdeeKHa6iLZHckmyVYXmXn15ptvIp4Zw1iuOmp22B+LiIgoToKdp556CsOHD1crHhs1MZLRkRYORmEx1WS3pqq9UzI7miZRXLXHc4yaHc7GIiIiiu0wlgwTSWBz0UUXqRlPssmxFBbLY1Q7u38Yyy1BTi1FylWzsRjsEBERxXxRwVatWgXWxqH6sVvT1N4pwY40A7VWX+eHw1hERERxFOzIVO+XX35Z9aUSXbt2xQ033KBaL9DhMzsuGb3yVPXlCu6NJZweHyrdXqTY9C7pREREFOVhrFWrVql2EVK7YwxjyVR0uW/NmjVHeTqJy+HP5Lgks1NLf6xMh1V1PxfM7hAREcUwsyNr48i6OjIjS3pQGSsW/+lPf1Jr4ixatCiMp5c47GZ70DBWZa2zzrJSrDhY7lbBTn5W1dR6IiIiimKwI5md4EBHvYjVqtbDqW39HTpkUUGV2am7P5YEO5yRRUREFMNhLGn0uW3bthr3b9++HZmZmeE6r4TjMNbZOVywk6YHRBzGIiIiimGwc+WVV2LUqFGYNWuWCnBkmzlzphrGCm4hQQ3L7AgGO0RERDEcxnriiSdUfcl1112nanWEzWZTjTkfffTRMJ5aYgY7ddXsCAY7REREcRDs2O12PPPMM6rD+a+//qruk5lYaWn6OjJ0+AJldx2zsYJXUS4qZ+dzIiKimA1jGSS4kU7nbdu2xWeffRZYc4cOX7OjMju1rLMjmNkhIiKKg2DniiuuwPPPP6+OKyoq1Awsua979+7sjXUYNoutalFBWUG5Fgx2iIiI4iDYkXV0Tj/9dHX8/vvvQ9M0taLys88+i4ceeigCp5iIs7EOn9kpZLBDREQUu2CnqKgIjRs3Vsdz585VHdBlSOuCCy5QDUKpPrOx6sjs+FtGMLNDREQUw2CndevWWLp0KcrKylSwM2TIEHX/wYMHkZLCVX/rwtlYREREx8hsLGkJMWLECGRkZKji5LPOOiswvCUFy1Sf2VhHCHa4gjIREVHsgp1bbrkFffv2VYsJnnPOOTCb9eRQhw4dWLNT79lYtQc7OUHDWFILJesZERERUZSDHSEzsA7tgyU1O1TPmp0jzMby+DSUu7xIdzTo00NERERB6vXbdOzYsZgwYQLS09PV8eFMnjy5Pi+ZtMGODGP53JW1Fkul2iywWUxwezWV3WGwQ0REdPTq9dv0m2++gdvtDhzXhcMuR67ZEW5POfRBrZrXT7I7+0pdqvN5i5zUqJ4jERFR0gY7CxYsqPWYQq/ZEU5PZa3BjjCCHc7IIiIiinG7CGF0Pacjs5qr4kpXHevsCE4/JyIiinGwI53O77vvPmRnZ6Ndu3Zqk+N77703MNRFtQ9ROUzWEIIdNgMlIiIKh5ArYG+//Xa89957mDRpEvr376/uk0UG77//fuzfvx9TpkwJy4klIrvZCqfXA5e39nYRIidNr+1hZoeIiChGwc6bb76JmTNn4rzzzgvcJ01AZWXlq666isHOkYqUvZVw1tEbS3AYi4iIKMbDWA6HQw1dHap9+/aw26tmHFFNdrMeyLgPk9nJMpqBchVlIiKi2AQ7t912m1pzx+ms+oUtxw8//LB6jOrmMPpj+equx2Fmh4iIKMbDWLLOzvz589GqVSv06NFD3fftt9/C5XJh0KBBuOyyywLPldoeqmIzVlH21B3s5DDYISIiim2wk5OTg+HDh1e7T+p1qP5r7bh87iNmdooZ7BAREcUm2Hn11VfD885JyG40A9U8gM8H+JuoBsv2NwMtZLBDREQUu0UFZa2dzz//HC+++CJKSkrUfTt37kRpaWl4zipB2a0pVc1A6yhS5jAWERFRjDM7W7duxbnnnott27apwuRzzjkHmZmZeOyxx9TtqVOnhvkUE4fDmlq987kt9bDDWD6fBrOZ/caIiIiimtkZM2YMevfujYMHDyI1teqX9aWXXqoKl6ludqujKtjxVB526rlPA0qcnqieHxERUSIKObOzePFiLFmypMaaOrL2zo4dO8J5bgnHbkw9P0ywk2KzIMVmRqXbp7I7RqaHiIiIopTZ8fl88Hq9Ne7/7bff1HAWHWEFZVlUUA1j1R7sCK61Q0REFMNgZ8iQIXj66aerNbiUwuTx48fj/PPPD+OpJXpm58jNQLmKMhERUQyGsZ588kkMHToUXbp0QWVlJa6++mr8/PPPyMvLw1tvvRWGU0r8YMclNcfsj0VERBSfwY6snCwrJs+aNUvtJaszatQojBgxolrBMh1mUUFjNlYdslPZ+ZyIiChmwY76IKtVBTeyUeg1O/psLGZ2iIiI4nZRQYpSzU5F3T20iIiIqH4Y7MSkZufws7Fy0kLvj3WwzIUzH1+ACXN+CMOZEhERJQ4GO7Gq2aljnZ2GDmMt37wfW/eX46Nvd4bhTImIiJI02JH1dRYtWoTCwsLInVECs5ltIQU7oUw9/3VvmdrvLXXC7fUd9bkSERElZbBjsVjUOjvSKoLiK7Pz6169CaumAXtL6i5+JiIiSjYhD2N169YNmzZtiszZJNM6O4dbQTmtIcGOntkRBcV1vzYREVGyCTnYeeihh/DXv/4Vc+bMwa5du1BcXFxto/DNxiqq5zCWpmnYtEfP7IjdRQx2iIiIGrzOjtES4uKLL1atIoJ/4crt2vpmUS2zseqxzo50Pff6NFjMVde5NjJsFdwhfReDHSIiooYHOwsWLAj1QyjkFZSrOp3L9PNG6dU7zB9uCEvs5jAWERFRw4OdM888M9QPoVpXUK47ILFZzEi3W1Dm8qq6nSMHO1VDWII1O0REREe5zs7ixYtxzTXXYMCAAdixY4e679///je++uqrhrxckg5jHT4gqVpF+ch1O0aw0ywrRe05jEVERHQUwc67776rup5L0881a9bA6dRrT4qKivDII4+E+nJJpb4rKIusEKafb/IPYw04LlftOYxFRER0lLOxpk6dipdeegk2W1VtycCBA1XwQ0eu2dFnYx0+IDFaRhSFkNkZ2DFP7QuKKlXBOBERETUg2Nm4cSPOOOOMGvdnZ2dzZeV6rqDsNpmgHWbqefXp54dvBlrp9mJHof5a/TvqmR2nx8eO6URERA0Ndpo1a4Zffvmlxv1Sr9OhQ4dQXy4pMzvCdYRhrPquorx5X5laNVkyQc2zU9DYX8zMuh0iIqIGBjs33ngjxowZg+XLl6t1dXbu3Ik33nhDLTR48803h/pySVmzI5xHHMay1yvYMYawOjbJUJ+PfH+RMmdkERERNXDq+d133w2fz4dBgwahvLxcDWk5HA4V7Nx+++2hvlxSDmMJl/fw/avq2wz01z16cXKHvHS1b5blwIZdXEWZiIiowcGOZA/++c9/4m9/+5saziotLUWXLl2QkZER6kslHbl2DrMNTp/7iMFOfWdjBTI7TfXr3yw7Ve05jEVERNTAYMdgt9uRmZmpNgY69WevZ7BT35qdTfuqhrGC19rh9HMiIqIG1ux4PB7cd999avZVu3bt1CbH9957L9xuzgA6Ersx/fwIwU5OPYIdn0+rGsZq4h/GytZfnzU7REREDczsSF3Oe++9h0mTJqF///7qvqVLl+L+++/H/v37MWXKlFBfMimLlN2aD/B6AIu1wZkdCWgq3F5YzSa0aZxWbRhL1tohIiKiBgQ7b775JmbOnInzzjsvcF/37t3RunVrXHXVVQx2Ql1Y0JLR4GDHWDm5bW6a6qcVPIzFzA4REVEDh7Fk5pUMXR2qffv2qo6HDs8W6HwuY4LOI66gXO7ywuXxHbY4uYO/Xic42JFZXLLgIBERUbILOdi57bbbMGHChEBPLCHHDz/8sHqM6pfZ0ZuB1r2KcmZK1TT1urI7wWvsGLJSrUix6Z9WFikTERE1YBjrm2++wfz589GqVSv06NFD3fftt9/C5XKptXcuu+yywHOltoca1gzUYjYhM8WKkkqPCnaaZFatvlwz2NGLk43p7c2zU9XKyjL9vG1u1WNERETJKOTMTk5ODoYPH44LL7xQ1enIJscS5MisrOAtFI8++qj6RX3HHXcE7qusrMStt96K3NxcNb1d3nf37t3VPm7btm244IILkJaWhqZNm6r1f2TGWLwHO+FoBmrU7Bhr7Bjys/TAiJkdIiKiBmR2Xn311bCfxMqVK/Hiiy+qQudgd955Jz7++GPMnj1bBU8yTCZB1ddff60e93q9KtCRfl1LlizBrl27cN1116lu7I888gjiejZWPYIdKVLejgoUVdRsBlrq9AQWDuyYVz3YCRQpc0YWERFR6JmdcJMVmEeMGIGXXnoJjRo1CtxfVFSEl19+GZMnT8bZZ5+NXr16qUBLgpply5ap53z22Wf44Ycf8J///Ac9e/ZUM8SknuiFF15Qw2p1kRqj4uLialtMZmO569n5vJbMzmZ/Vicvw45sfwbIwFWUiYiI4ijYkWEqyc4MHjy42v2rV69WixQG39+5c2e0adNGresjZH/SSSchPz8/8JyhQ4eq4GX9+vV1vufEiROrDbfJUFy0+2PpBcr1XEW5lv5YxsrJwTOxDNIfS3AYi4iIKMbBjqzXs2bNGhV8HKqgoEBNZZcaoWAS2MhjxnOCAx3jceOxutxzzz0qc2Rs27dvR7zNxhLZqUbn85o1SL/uqVmcbGiWzbV2iIiIjro31tGSAGPMmDGYN28eUlL0X87RImsFyRbz2Vj1zOwU1lKz86tRnFxLZiff6I/FYSwiIqLwZHYKCwtD/hgZptqzZw9OOeUUWK1WtS1cuBDPPvusOpYMjdTdHPraMhtLCpKF7A+dnWXcNp4Tv7OxcFQ1O7WtsWOQqedid4kTXp8WlvMmIiJKmmDnsccew6xZswK3r7jiCjU1vGXLlmq9nfqSNXnWrVuHtWvXBrbevXurYmXjWGZVyZo+ho0bN6qp5kZPLtnLa0jQZJBMUVZWFrp06YJ4ZDcHZ3bqN/W8+JBgRwIYWUcnuAFoMClaNpv05+0vPXz2iIiIKNGFPIw1depUvPHGG4HAQrZPP/0Ub7/9tlrjRmZI1UdmZia6detW7b709HQVOBn3jxo1CmPHjkXjxo1VACNNSCXAOfXUU9XjQ4YMUUHNtddeqxqTSp2OdF+XoudYDVOFVrNz5KnnRuuHYDsLK+D0+GC3mNGqkd4ANJjVYlaLEO4udqq6nab+YS0iIqJkFHKwIwGFMXtpzpw5KrMjQYf0y+rXr19YT+6pp56C2WxWiwnKdHGZafWvf/0r8LjFYlHncPPNN6sgSIKlkSNH4sEHH0S8slls9VpB+XDDWL/4h7Da56WrlZZrI9PPJdiR6efdW4Xp5ImIiJIh2JG1cKS4WAKeuXPn4qGHHlL3a5qmFvk7Gl9++WW121K4LGvmyFaXtm3b4pNPPsEx2/W8AcFO1crJdbeCkOnnMqjI6edERJTsQg52ZAXjq6++Gp06dcL+/fvVQn5Gz6zjjjsuEueYUIyanfquoCwKDwl2At3OD1k5ORhXUSYiImpgsCNDSzJkJdkdqZORnlVCWjXccsstob5c0qnWG+tIs7H8Bcoujw+Vbi9SbJbqa+wcJrOTz7V2iIiIGhbsyAypv/71rzXulz5WFN51djLsVjWrSmaPy1CWEexs8s/Eqm3auaG5Eewws0NEREmuQYsKyhTw5557Dhs2bFC3TzzxRDVT6oQTTgj3+SWcqtlYOOIKymazSQ1lHSx3q2BHFguU/d4SZ6BAuS7GwoLM7BARUbILeZ2dd999V00Nl0UBe/TooTZp+SD3yWMUwjDWETI7tU0/3+Sv18nPciAzpXoD0LpqdqR4nIiIKFmFnNm56667VG+pQ6d3jx8/Xj0m08SpnsNYR6jZqW1G1uHaRNTWH6vc5UWJ04OswwRGREREiSzkzI4UIl933XU17r/mmmvUYxS+FZRF1iHBjpHZqW3l5GBpdiuyUvRYlj2yiIgomYUc7Jx11llYvHhxjfu/+uornH766eE6r4QVygrKIidND44Ky11H7Il1KHY/JyIiasAw1sUXX4y///3vqmbHaNuwbNkyzJ49Gw888AA+/PDDas+lww1jHTkIyU61VuuPVd9hLKNI+afdpZyRRURESS3kYMdYS0faNgS3bgh+TJhMpqNeUTnxp56HVrPj8fqwdb+xevKRgx1OPyciImpAsOPz+SJzJkmieruII8/Gykn1D2NVuLH9YAXcXg0pNjOa16O5Z2BGFoexiIgoiYVcsxOsspK/RENlM9sC7SK0EGdjGSsnS5sIWYPnSIxVlNkfi4iIklnIwY4MTU2YMAEtW7ZUrSI2bdqk7r/vvvvw8ssvR+IcEzKzI1z1yOwEz8YKFCfXYwgreBhLOp8TERElq5CDnYcffhivvfaa6otlt+tDLEIWFZw+fXq4zy9ha3aEy1sp7eIP+/yctKpgx+h23uEwKyfXtooyMztERJTMQg52Xn/9dUybNg0jRoyAxaL3ahKykvKPP/4Y7vNL2GEs4YQGeKt3NK9zGKs89MyOUbOzr9SlmokSERElo5CDnR07duC4446rtXDZ7T78L27SZ6kZCwu667HWTrWancAaO/XL7DROt8Nu0T/Fe0qY3SEiouQUcrDTpUuXWhcVfOedd3DyySeH67ySaEZW/YIdj09TDUGP1AD00MAqP1t/L04/JyKiZBXy1PNx48Zh5MiRKsMj2Zz33ntPdUGX4a05c+ZE5iwTjM1iA9z1W0U5zW6BzWJSU85Fy5xU1QqivmQoa/uBCk4/JyKipBVyZueSSy7BRx99hM8//xzp6ekq+NmwYYO675xzzonMWSZyy4gjrKIs2Rkju1Ofnlh1FSkzs0NERMkq5MyOkB5Y8+bNC//ZJIlQV1GW6edSZFzfNhHBuIoyERElu5AzOx06dMD+/ftr3F9YWKgeo/oHO05ZF7BeqyhXZXbqW5xcI7PDYSwiIkpSIQc7W7ZsqbXnldPpVHU8dGTVZmOFsIpyQzI7RudzrrVDRETJqt7DWMHdzP/3v/8hOzs7cFuCn/nz56Ndu3bhP8Mkn41VI9ip5xo7BvbHIiKiZFfvYGfYsGGBglmZjRXMZrOpQOfJJ58M/xkm6mysQM1O/YOddLsFTTOr2k2ElNkpckLTNPX5IyIiSibWULudt2/fHitXrkReXl4kzyuhhTIbS2Sn2QNZnVCDlaaZerDj8vpwoMyF3IzQgiUiIqKkq9nZvHkzA50w1ezUN7PTulGq2ndrmR36e1nNyMvQ349DWURElIzqHewsXbq0xqKBspCgZHqaNm2K0aNHqyJlCmU2Vv2CnUt6tsTUa07BXUNPaND7GUNZnH5ORETJqN7BzoMPPoj169cHbq9btw6jRo3C4MGDcffdd6tFBSdOnBip80zcdXbqMRtLsjPndmuOHP9wVqhYpExERMms3sHO2rVrMWjQoMDtmTNnol+/fnjppZcwduxYPPvss3j77bcjdZ6JW7NTj3V2jpax1s5uZnaIiCgJ1TvYOXjwIPLz8wO3Fy5ciPPOOy9wu0+fPti+fXv4zzAB2cy2oEUFj5zZOVrGKsq7GOwQEVESqnewI4GOFCcLl8uFNWvW4NRTTw08XlJSoqagU/xmdjiMRUREyajewc7555+vanMWL16Me+65B2lpaapHluG7775Dx44dI3WeSV2zc7S4ijIRESWzeq+zM2HCBFx22WU488wzkZGRgRkzZsBuryqYfeWVVzBkyJBInWcCNwKNfAASKFDmMBYRESWhegc7srbOokWLUFRUpIIdi8VS7fHZs2er+yn87SLCldkprvSg3OVBmr1Bze6JiIiSY1FB6Yl1aKAjGjduXC3TQ/VtBBr5YCczxaZaTQhmd4iIKNmEHOxQ9BcVDId8Y2FB1u0QEVGSYbCTBDU7wdPPmdkhIqJkw2AnplPPEZXZWILTz4mIKFkx2Il5Zic6/cSMGVlcRZmIiJINg52Y1+xEJ7MTaAbKzA4RESUZBjtJMBtLcK0dIiJKVgx2kmCdHcHMDhERJSsGOzFgs9iiPhvLyOzsLXHC4/VF5T2JiIjiAYOdmDcCrQQ0LeLvmZvhgNVsgk8D9pZGpyiaiIgoHjDYiWHNjgsy9xxRmZFlMZvQNFMPsli3Q0REyYTBTiynnptNUDmdKM3IMlZRZvdzIiJKJgx2YhjsCLf8L8pr7TCzQ0REyYTBTgxrdgIzsqK0irIxI2sXMztERJREGOzEgM2sz8YSXEWZiIgoshjsxIDJZKoqUuYqykRERBHFYCce+mMdg6soS5Hze2t+Q6XbG4YzIyIiihxrBF+bjhTsuGO3irIEKSk2S4NeZ8GPezD27bU4WO7GgTIX/nR6hzCfKRERUfgwsxMXnc+jE+zkZ6Ug02FFpduHC55djDXbDob08W6vD49++iP++NpKFeiIhT/tjdDZEhERhQeDnXhZRTkKJJPz/IhT0CTTgV/3luHyKUvwyCcb6jUUtauoAldNW4apC39Vt8/t2kztV245AJeH7SeIiCh+MdiJ8YwspyyiHKWaHXHm8U0w784zcNnJLVXriGmLNuH8ZxZj9dYDdX7Mgo171HNWbT2IDIcVL1x9CqZccwpy0+0qS7R2e2HUzp+IiChUDHZinNlxR3E2liEnzY7JV/bE9Ot6qxYSm/aV4fKpS/HQnB9Q4arK8kjD0Mfm/og/vqoPW3VtkYU5t5+GC7o3VzPK+nfMVc9b8uu+qJ4/ERFRKBjsxLhmRy9Qjk1jzsFd8jHvzjNxea9Wqhfp9K824/xnF6uhKTVs9dIyTPlSH7a6rn9bvHvzALTLSw98/ICOeWq/5Nf9MTl/IiKi+uBsrLiYeh7dzE6w7DQbnvh9D1xwUnPc/d532LyvDFe8uBQZditKnB41bPXY8O4qm3OoAf7MzjfbDqqMUKq9YbO7iIiIIomZnSQqUD6c33Vuis/uPBNX9NazPBLoBA9b1aZtbhpaZKfA7dWw6jA1P0RERLHEzE6MVF9BOfbBjshOtWHS5T0wrGdL/LS7BH/o2+awa/HodTt5eHfNb2oo6/ROTaJ6vkRERPXBzE481OxEcTZWfQw4Lg/XD2xfr0UHjaEs1u0QEVG8YrCTRIsKRoIxI2vdb4UortQXGiQiIoonDHZiJN5qdhqqRU4q2uelqzV7Vmxi3Q4REcUfBjsxXlTQpRYVjN1srHCoWm+HQ1lERBR/GOzEOLMTy3V2wqWqboeLCxIRUfyJabAzceJE9OnTB5mZmWjatCmGDRuGjRs3VntOZWUlbr31VuTm5iIjIwPDhw/H7t27qz1n27ZtuOCCC5CWlqZe529/+xs8Hg+OhZqdWKygHG6ndtCDnR8LSrC/9NgO3IiIKPHENNhZuHChCmSWLVuGefPmwe12Y8iQISgrKws8584778RHH32E2bNnq+fv3LkTl112WeBxr9erAh2Xy4UlS5ZgxowZeO211zBu3DjEs3hYQTlc8jIc6NwsUx0vY90OERHFmZiuszN37txqtyVIkczM6tWrccYZZ6CoqAgvv/wy3nzzTZx99tnqOa+++ipOPPFEFSCdeuqp+Oyzz/DDDz/g888/R35+Pnr27IkJEybg73//O+6//37Y7XpQEczpdKrNUFxcjJius3OM1+wYdTuS2ZGhrLoWISQiIkKy1+xIcCMaN26s9hL0SLZn8ODBged07twZbdq0wdKlS9Vt2Z900kkq0DEMHTpUBTDr16+vc/gsOzs7sLVu3RrRliizsQ7tk7WURcpERBRn4ibY8fl8uOOOOzBw4EB069ZN3VdQUKAyMzk5OdWeK4GNPGY8JzjQMR43HqvNPffcowIrY9u+fTuiLVHW2TH0bd8YZhNUB3VpIkpERBQv4ibYkdqd77//HjNnzoz4ezkcDmRlZVXboi2eV1BuaKuJk1pmq2Nmd4iIKJ7ERbBz2223Yc6cOViwYAFatWoVuL9Zs2aq8LiwsLDa82U2ljxmPOfQ2VnGbeM58T0bCwmR2RHSJ0twvR0iIoonMQ12NE1Tgc7777+PL774Au3bt6/2eK9evWCz2TB//vzAfTI1Xaaa9+/fX92W/bp167Bnz57Ac2Rml2RrunTpgmNjnZ3ECHaM9XYksyOfWyIiIiT7bCwZupKZVv/973/VWjtGjY0UDaempqr9qFGjMHbsWFW0LAHM7bffrgIcmYklZKq6BDXXXnstJk2apF7j3nvvVa8tw1Xxv4KyCfC6AJ8XMB+58WY8692uEWwWE3YUVmDbgXK0zU2P9SkRERHFNrMzZcoUVSB81llnoXnz5oFt1qxZgec89dRTuPDCC9VigjIdXYam3nvvvcDjFotFDYHJXoKga665Btdddx0efPBBxLNqs7HEMb7WjkizW3Fy60bqmENZREQUL2Ka2anPUEdKSgpeeOEFtdWlbdu2+OSTT3AsqVagLGQoy56GRFhvZ8WWAyrYuapvm1ifDhERUXwUKCejalPPRQIsLFi9bmcf63aIiCguMNiJkaoVlP2fggQpUu7ZJgcpNjP2lbrw857SWJ8OERERg53Y1+wgoYIdh9WCPu30FbCX/MIu6EREFHsMdmLEZqmajaUlULBj1O0IFikTEVE8YLAT48yOcKv/JU6wY/TJWrZpP7w+1u0QEVFsMdiJcYFy1cKCiVGgLLq1yEJmihXFlR78sDP6HeWJiIiCMdiJcYFyVTPQY3+dHYPVYka/9sZQVnjqdjxeH1weX1hei4iIkguDnRgxmUxBM7KkGWjiZHaCp6CHo25n7fZCnD5pAc6YtADrfisKw9kREVEyYbATL2vtJFBmRww4Tg92Vm45cFQZmf+u3YErX1yKXUWVKCiuxJXTlmLBxqo+aEREREfCYCdeVlFOoJodcXzTTOSm21Hu8uK736p3ra8Pn0/Dk59txJiZa+H0+DCoc1Oc3ilPvd6fZqzCrJXbInLeRESUeBjsxEGw45Zgp3QvEonZbMKp/qGs/yzbiqIKNeesXsqcHtz8xmo898Uv6vafz+yAadf1xivX98HwU1qpGV5/f3cdnpr3E1dpJiKiI2KwEwfTz1VmZ/tyJJpzuzZT+w/W7sTAR7/AxE82YHfx4afY/3awHJdPXYr/rd8Nu8WMJ3/fA/ecdyIsZhNsFjOe+H133H72ceq5z8z/GXe98x3cXhYuExFRnDYCTXY2s7GwoPyWXwl4PYAlcT4lF3ZvrrIw//ryF/y0uxQvLtqEV7/egktPbonRZ3ZAxyYZ1Z6/eusB/Pnfq1WribwMO168tjd6tdW7qAcXdv9lyAlonp2Kez9Yh9mrf8PuEif+NeIUZDgS59oREVH4MLMTDy0j7BmAqxTYvQ6JRAKTYSe3xNwxZ+Dlkb3Rp10juLw+zFq1HYMnL8RN/16tZlqJd1b/hqumLVeBzonNs/Df206rEegEu7pfG7x0XW+k2ixY9NNeVcS85whZIyIiSk4MduJhNlaT4/U7ti1DIpL6nUEn5mP2TQPwzk39MfjEppBSm7nrCzDsha9x7tOL8NfZ36pAaGjXfPWcljmpR3xdec23Rp+qCqHX7yzGpf9agl/YfJSIiA7BYCceZmPlddTv2LoEia53u8aYPrIPPrvzDFVsbDWb8GNBiXrs/84+DlNG9EJ6CMNRPVvn4L1bBqBdbhp2FFZg+JQl+HTdrgj+C4iI6FjDYCeGAosKNmpfldlJktlFx+dn4skremDhXb/DmEGdMP263hg75ASVBQpV29x0vHvzAJzcJkfN+rr5jTW4c9bakGaAERFR4mKwEw81O1nNAMnylO0BDmxCMpHhqjvPOR6Du+Qf1evkZjgwa3R/3Pq7jpB46f1vdmDoU4uw+OfEmtJPREShY7ATQzaLfzYWNKBlr6QZyooUu9WMvw3trGqDZFhLVly+9uUVGP/f71Hh8sb69IiIKEYY7MTDOjteJ9Cmf0IXKUeTzOL6ZMzpuPbUtur2jKVbccGzi/HNtoOxPjUiIooBBjtxULPj9rqDgh1mdsIhzW7FhGHdMOOGvsjPcmDTvjJVvCwtKMLRPX1/qRNXTF2Ke977jt3YiYjiHIOdeJiNJZmd1n1lZRq9Zqdkd6xPLWGceXwTfHbHmRjWswV8GlQLikv/9TV2FR1dL7Jn5/+MFVsO4K0V23HLG6vh9HCYjIgoXjHYiYd1dnwuIDUHyO+qP7BtaWxPLMFkp9nw9B9OxgtXn4KcNJtak+e+D9Y3+PW27S/Hmyv0RqQ2iwmfb9ijFkisdDPgISKKRwx24mE2ltel38G6nYi6oHtzzP5zf7W2z+cbdquVlxviyXkb4fZqOOP4Jnj1+r5IsZmxYONejGbAQ0QUlxjsxMswlmhzqr5n3U7EdMrPxHX926njB+f8EHIT0e93FOG/a3eq47uGnoDTOuWpgMdoW/GnGas484uIKM4w2ImHRQWNzE7bAfq+YB1QWRzDM0tsYwZ3QuN0u2ot8e+lW0P62En/26j2l/RsgW4ts9Vx/465qhA6zW7BV7/swx9fW4Fylyci505ERKFjsBMPNTtGsJPVAshpC2g+vQs6RUR2qg1/HXKCOn7q85/UzKr6WPLLPpW9kTqdv5yjf7yhb/vGeP2Gvqrz+rJNB3D9KytR6mTAQ0QUDxjsxEPNjhQoGwJ1OyxSjqQr+7RGl+ZZKKn04Ml5Px3x+Zqm4bG5P6rjq/u2QZvctFr7fr0+qi8yHVY1U+v6V1agpJItK4iIYo3BThysoByo2RFtWaQcDRazCeMv6qKO31qxDet3Fh32+Z9+X4BvfytSQ1W3nd2pzued0qYR/vOnfshKsWLV1oO47pUVKGbAQ0QUUwx24iCzoxYVPDSzI8NYnqCMD4Vdvw65aoaW9F594KMfVPamNlLE/IS/VufG0zugSab+eatLj9Y5ePPGU9Vw2TfbCnHN9OX1HiojIqLwY7ATBwXK1TI7eccDqY0BTyWw69vYnVyS+Mf5J8JhNWPF5gP4ZF1Brc95e9V2tQJzbrodN57RoV6vK8XLb97YD43SbPjutyJcNmUJNu8rC/PZExFRfTDYiZdFBQ0mE1tHRLnr+k1ndlTHj3yyoca0cZlV9cznP6vj284+ThUg11fXFtl45+YBaNUoFVv3l6t2FWvYn4uIKOoY7MTTooKH1u1sZZFyNEiw0yI7BTsKKzBt0aZqj7369RbsKXGideNUXN2vTciv3bFJBt67ZQBOapmNA2UuXDVtGf63vvYMEhERRQaDnXiaem4wMjvblwE+NpmMtFS7Bfecf6I6nrLwFxX0iINlLkz98ld1LFPNHVZLg16/aWYKZo4+Fb87oQmcHh9u+s9qzFiyJYz/AiIiOhwGO/G0grKheQ/AlgZUHAT26YWxFFkXdm+Ovu0ao9Ltw6Of6lPM//XlLyhxenBi8yxc3KPFUb1+usOKl67rjav6tlEF0eM/XK+GzXzSnZSIiCKKwU4cFChXm40lZEp6q976MdfbiQqTyYRxF3VRJVMffbsTH3yzAzOW6Ksr33XuCTCbTUf9HlaLGY9c2g1/G6ovSChDZv8385s6O6aXOT1qIUPpsC5T2M9+4kuM/+/3qmVFXTPHiIiopvpXW1LEanZqZHaMoazNi/S6nd43RP/kkpDMoPpDnzZq3Z07Zq1V953aoTHOOr5JWIOqW393HJpnp+Cud77DnO92qZqgl67tjTKXB6u3HlTbqq0HsGFXCbyHZH5kVtiMpVvRuVkmft+7NYb1bIHcjMNPhSciSnYMduJgUUGZjSV/qcsvwgB2QI+Jvw45HnO+26lWVhZ/P7dz9c9LmFx2SivkZ6Xgpn+vVtPe+z7yuarnOZQUTvdq1xi92zZCfpYDH323C/PW78aPBSWYMOcHPPrpBgzqnI/f926FM49vorJHRERUHYOdOMjsCLfPHajhUVr1AUwWoGgbUPQbkN0qNieZZCRLIn2zpKZG6nhObtMoYu818Lg8vH1Tf/zx1ZUoKK5UqzpLC4tebRsFthY5qdU+5txuzVFY7sKH3+7E7FW/Yd2OIsxdX6A2WezwspNb4obT2qtAioiIdCaNg/8oLi5GdnY2ioqKkJWVFbX3leGr3v/Ra3OWXrUUGfaM6k+Ydhaw8xvgsulA999H7bwIWPdbETrlZyDF1rAZWKGQWV+/7C1VgY4UModiw65iFfR8sHaHmtouZPHDZ/5wMk7rlBehMyYiOrZ+fzPnHQcFyoet2xEsUo66k1plRyXQEY3S7ejTrnHIgY6QmWJSWL3snkGYek0vdXt/mQvXvrIcz83/mbO9iIgY7MSW1ILYzLbAMFYNDHaonuxWM87t1gzv3zIAV/Rupaa3Szf3UTNWqmEvIqJkxmAn3mdkiT0/6GvuEB2BZKMmXd4Dk4Z3Vz2/Fmzciwue/Qrf/VYY61MjIooZBjvxuoqyyGgC5B6nH29bHuUzo2PZFX1aqzYVbXPT1IrQl09Zin8v28r1eYgoKTHYiedgR3AoixpIGpF+dPtpGNIlHy6vD/d98D3unLVWNTclIkomDHbipEi5WufzYAx26Chkpdjw4rW98I/zO6up7R+s3YlLnv8ayzfth6uWdX2IiBIR19mJ1/5Yh3ZA37EGcFcAturrrhDVpxB+9Bkd0bN1I9z25hr8vKcUV05bpmp6erTKQa92jdCrjb6uj8wMIyJKNAx24n0Yq1F7ICMfKN2tBzztBkb3BClh9G3fGHP+7zQ8/PEGLPppLw6Wu7FiywG1GTo2SUfvto1V4CPT79s0TmvQlHgionjCn2JxMhurzmBHWhXIUNYPHwBfTgQ6XwDkHa9vsqpyBFoZUOJqmpmiFhyUQmXps7V6i96Ha9XWg9i0twy/+rdZq7YHPkYWKWzdOE0FPq0bp/r3+u3m2alqeIyIKJ4x2ImTmp06h7FEx9/pwc6WxfpmsKUDeZ2AJifowY/sO5wFODKjcOZ0rA9tdWySoTaZuSVkBeY1qgmpNCM9oIa7CsvdapFC2dZurzl9PdVmURmj047Lw4DjcnFis6ywdIgnIgonBjvxPowlel4DpGQDBeuAvRuBfT8BBzYB7jJg11p9M+S0Bf74CXtpUcgap9sxuEu+2gzFlW5sP1Cutm2BrULd/u1gOSrcXiz8aa/ajNcY0DFXBT/S+0syQEREscZg51gIdixWoOul+mbwuvWAxwh+ZNu0ECjcCrx2oR7wZLWIwr+AEn02l0xhl+1QXp+Gn3aX4Otf9qlt+eYDKjs057tdahOyzo8EPhf1aIF+7RtHpIM8EdGRsBFoDBuBin9+9U98+OuH6NusL547+zmk2Y7iL+HC7cBr5wOF2/TFCK//GMhsFs7TJaqTTGWXoS4j+Plme6EKiAxS43N5r1YY3qsVWh7SzZ2IKJK/vxnsxDjYWbd3HUZ9NgoVngqc3PRkPD/oeWTZj+IcDkpm5wKgaDuQdwJw/Rwgo2k4T5moXkoq3Vix+QA+W78bc77biTKXV90vyR3J9kjgM7Rrs6g1XCWixMNg5xgJdsTaPWtxy/xbUOIqQefGnTF18FTkpuY2/AVleOvVC4CSnUDTLsDIOUD6Ubwe0VGSVZs/XVeA2au3Y9mmqqnumSlWXNyjhQp8js/PVAXPLHAmovpisHMMBTti44GNGD1vNA5UHkC7rHZ4achLaJZ+FENQ+38FXj0fKC0A8k8CRn4IpDUO5ykTNci2/eV4Z81veHf1b6pv16FkscM0u0UFPqmyt1uQZrMixW6BxEEyNObTNLUPbBrg8x9np9rQrWUWurXMRvdWOWjbOI0BFFGCYrBzjAU7YkvRFtw470YUlBWgRXoLTBsyDW2z2jb8Bff+pA9ple0BmvcArvsvkNoonKdM1GASnCzdtB9vr9qO/60vQKU7Mu0rMh1WFfjIIokqAGqZrQqnWSxNdOxjsHMMBjtiV+kuleHZUrwFuSm5KuA5vtHxDX/BPRv02Vnl+4AWpwDXfaBPYyeKs8DH6fGp4a5ylxeVbq/ay9T2Cpd+LI/JDyuLyQSrxQSzyaQWNDT2FjPU8Z5iJ9btKMJ3O4qwYVdxrT3AJAA6vlkmjs/PUMNnJ+Rnqtt5Gfoin0R0bGCwc4wGO2JfxT7cNO8mbDy4URUrTxk8Bd2bdG/4CxZ8D8y4EKg4CLTqC1z7HhcepKTg9vrU9PjvdxSpAGjdb0XYUFBSZxNUWSdIAiAJfjo2zVBDYul2qxpWS3NYkR68t1tht7KXMlEsMdg5hoMdUeQswq3zb8W3e79FqjUVz5/9PPo279vwF9z1LTDjIqCyCMhpow9rNWoXtLUHslsDVn3dH/myYJqfEjUA+nVvKX7aXYqfCkqwcXcJft5dgq0HyhHqT0ObxaRqiyTwUfVF6thfZ+QPiGS2WU6aTbXdkMxRboYduekO5GXYVeNVm6SkIky+n91e2Xxqc3nrMWSoAW6fpgLDwOb1qgxc1W2fyqbJv1V6qKXbrUh36P/uDIf8281R+TkS/O/zSAGXCSrbZ/Vn/tSedVuKXKOiCjdKKj0qgyqfR/mcOj01j2WTgD7N/3lNV3tr1edbjmM8qYDBzjEe7IhydznGLBiDZbuWqbYSPZv2hEm+i9V/+heX7I0fJnJsMVtgNVlhs9hgNVurjk1WWMsPwLbhI3i8TpSbzSgzmVBqNqPcbEKpyYwyuc9iUfd7TUA6LEg325Ahm8mGDIsDGWYH0i0OZFpTkWq2Q/N54PW54fW54PPqxz512wOfz6v2LpMJLrMJTtnLeixqr8EJTe1dmg8Ws1m9Xop/S7XYkWJxINWSghSrAymWFDVnudzrRIWcf2CrDDp2wqv5kGFNQZY1DZlqS1X7DGta4L50awqgeQGvB5rmVf8GzeeFz+cB/Mea5oFT86Lc50alz4MKzYNKnxsVPg8qNX1f4XOr6241mWExmWE1WQLHNtlDv62GWWCC/EozB/bVj+Vz5zWZ4DOZ4JG9DO3I50HulwX8TPAX5Xrg0eTaeuBR196/aV61+Xw+9d52swUpJiscZhscJitSzDbYzfpe7oPJAq/ZBA/k/RDYy/t5TJp6T7fPC4/P7d88QXv/psn105Di/1ypz53s/Z8v/XOXovq/WTQfzD4PLD4vLF6Pfux1V+29bv+/3QyP2QS37OW2Oh/92njkOqjfwfqPLC2w6fcEfpDJ15N6P696P+N9ZW+VvdejHrda7LBZU2C1ONQe5hSUeywoclpxoMKMgxWA2+OC11MJTX19u+DzOaH53DBpbljMHpjhgdtkgRNWOKHvXbI36XvZ5N8CzQKrZoLVZ1J7mwa12TUg02JBjs0Mh8UHt+aCx+SUV4UbcuxWm9vkgUc2+KDBDA0WaLCqvQ9W+DT9ts9kg08e8xdsy4F8juRrx6THADBB8//0MMOk6RuMzX+fHKvH5N3MHljggdnkhdkk/2YvTLI3yV5fTsCrWeDTrPDCCq9/71F7G8xmOyzy9Wdyw27ywCYbPPoxPPJMtbeYPPDqr65/HapjMzya/2tA7tdM0P9ZVf8udez/d+k/B6u+GORaQdP/1fp3mR58yd6svkflasF/JaGuqtkkV1S/unKlrWYNZosETz6YzRosJg0ms2wS6MnL+iCvJO8T+C73Xz95VblX0ywwqceDfm4H79U5Am744NI86mei/lXghUvz6vf7b3vlE2k2QZOf+2bNv/ev6aD/MIFPNp8JPh/g9Zogsa3Hq29eeVB9ruW85Bzlvc3+Y/128DFMHn0ze2AyuaH5j+U+41i+Uqywwab+b4Vdfuag6udPqmwWK2455y50atExrL8nGewkQLBj9My6a+Fd+GL7F7E+FSIiogZ78eTHMaD7uYjF72+2i4hz8lfx5LMmY/mu5Sh06o0Y9b9k9b9oDMZt+ete/up2B/0VHnwsm9lsRoYtA+m29KrNmo50jxPp5QeRUboPlvIDKPOUodRdgVJPOUq9FSj1VqLU6/RvLpRr8lpWlU0ym23qrzezpWpvtchfdTbYYVJ/wTo0H+w+fXP4vLD7PLB7ZdMzQ5VeyaBIJsWFSp+3Kpui+VAJybZoSAeQppn0TY5hQqrmv08yW7KYHTSUmHwoAVBq0lBs0vz3yW39PpjM/qyY/y89tddzLcZtB8xINZmRIu8hx0H7FOj3y99EXmjwSBodkhHR4Anaq2yE5oPPZNb/2oKxSRanai//NrPKSGh6RkKOJQth7P33SfbIoq65DRb/tbYaxxa7+nzIeTglM+XzolLzqgyVui1/Lco11Xwq+2GVzSd7yYDIsZ75UJvKfshftZKd0jNUcmyTDKHcVudgVdfLCa/KchnvVaF5UQmfeh+1hwaf2QqvWbJJFvhkb5K9GV65Luq2/rmzavpf2Wov7682X2BvlgxF4O90+WtN/+oPvk/+0tXfz1rtfQPvZZY8hQav1+3/3gj6XpHvn8Am19ssOQB/5k72cv0tai/XwSzHxufMnz3SNz2DZfK5VeZKshJOixlOk8Wf5ZRjPdspLYBlk39KGizqay4VFqSZZPP/VWyyIt1sV99LkomU7GnVpmf31P2S9ZNspVwWk/+vf/XXuX6ssgz+7LDkZOTrU75eVHZP8+/V/frXrnwvWCD/dlPQNdCvg1Uyl2Z9CM4rmVH1s8e/l0yjzwcvfHrGUX5WmSzq+0q+9+Q4eK/JXl1plWOo+l5Qx3LG/s+98fn3j5romW5/tkTtqoZTJAemvtfk56L/Z6Zx23hMfa2o70H9ugS+HwOPSfJE5YH8X2+H7o1cmfGassn3e9WxvLcvKPsoj1VlJfXzNPZWyM8dk3/Tj1OC7pNjm5yTT/NnL/VrJdfG7JO93CfZKp+6FOo91KdIUwku/f39m/xcUkOAsvdnyuT7QvP5z8enrpVNva/8/JZzQOA85GtRfgbaJNtmMqPcZEJF0FYO2QOyuITsKwG0zKnquxdtDHaOAfLDdUDLAVF/3yZRf0ciIqLwS5ipBC+88ALatWuHlJQU9OvXDytWrIj1KREREVEcSIhgZ9asWRg7dizGjx+PNWvWoEePHhg6dCj27NkT61MjIiKiGEuIYGfy5Mm48cYb8cc//hFdunTB1KlTkZaWhldeeSXWp0ZEREQxdswHOy6XC6tXr8bgwYMD90kBrtxeunRprR/jdDpVBXfwRkRERInpmA929u3bB6/Xi/z86lXecrugoKDWj5k4caKaqmZsrVu3jtLZEhERUbQd88FOQ9xzzz1qTr6xbd++PdanRERERBFyzE89z8vLg8Viwe7du6vdL7ebNWtW68c4HA61ERERUeI75jM7drsdvXr1wvz58wP3yZL5crt///4xPTciIiKKvWM+syNk2vnIkSPRu3dv9O3bF08//TTKysrU7CwiIiJKbgkR7Fx55ZXYu3cvxo0bp4qSe/bsiblz59YoWiYiIqLkw0agcd4IlIiIiI7u9/cxX7NDREREdDgMdoiIiCihMdghIiKihJYQBcpHyyhbYtsIIiKiY4fxe/tI5ccMdgCUlJSoPdtGEBERHZu/x6VQuS6cjeVfhHDnzp3IzMyEyWQKa8QpAZS0o+Asr8jj9Y4uXu/o4vWOLl7vY+N6SwgjgU6LFi1UE/C6MLPj75LeqlWriL2+fOL4zRI9vN7RxesdXbze0cXrHf/X+3AZHQMLlImIiCihMdghIiKihMZgJ4Kks/r48ePZYT1KeL2ji9c7uni9o4vXO7GuNwuUiYiIKKExs0NEREQJjcEOERERJTQGO0RERJTQGOwQERFRQmOwE0EvvPAC2rVrh5SUFPTr1w8rVqyI9SklhEWLFuGiiy5SK2bKitcffPBBtcel5n7cuHFo3rw5UlNTMXjwYPz8888xO99j2cSJE9GnTx+1unjTpk0xbNgwbNy4sdpzKisrceuttyI3NxcZGRkYPnw4du/eHbNzPtZNmTIF3bt3Dyyu1r9/f3z66aeBx3m9I+fRRx9VP1PuuOOOwH283uF1//33q2scvHXu3Dni15vBToTMmjULY8eOVVPp1qxZgx49emDo0KHYs2dPrE/tmFdWVqaupwSTtZk0aRKeffZZTJ06FcuXL0d6erq69vJNRKFZuHCh+sGzbNkyzJs3D263G0OGDFGfA8Odd96Jjz76CLNnz1bPl9Yrl112WUzP+1gmq7nLL93Vq1dj1apVOPvss3HJJZdg/fr16nFe78hYuXIlXnzxRRVoBuP1Dr+uXbti165dge2rr76K/PWWqecUfn379tVuvfXWwG2v16u1aNFCmzhxYkzPK9HIl/D7778fuO3z+bRmzZppjz/+eOC+wsJCzeFwaG+99VaMzjJx7NmzR13zhQsXBq6tzWbTZs+eHXjOhg0b1HOWLl0awzNNLI0aNdKmT5/O6x0hJSUlWqdOnbR58+ZpZ555pjZmzBh1P693+I0fP17r0aNHrY9F8nozsxMBLpdL/VUmwyfB/bfk9tKlS2N6bolu8+bNKCgoqHbtpW+KDCPy2h+9oqIitW/cuLHay9e5ZHuCr7ekpNu0acPrHQZerxczZ85UmTQZzuL1jgzJXl5wwQXVrqvg9Y4MKSuQMoQOHTpgxIgR2LZtW8SvNxuBRsC+ffvUD6n8/Pxq98vtH3/8MWbnlQwk0BG1XXvjMWoYn8+nahkGDhyIbt26qfvkmtrtduTk5FR7Lq/30Vm3bp0KbmToVeoW3n//fXTp0gVr167l9Q4zCSal1ECGsQ7Fr+/wkz88X3vtNZxwwglqCOuBBx7A6aefju+//z6i15vBDhHV+69f+YEUPL5OkSG/CCSwkUzaO++8g5EjR6r6BQqv7du3Y8yYMaoeTSaSUOSdd955gWOpj5Lgp23btnj77bfVhJJI4TBWBOTl5cFisdSoIJfbzZo1i9l5JQPj+vLah9dtt92GOXPmYMGCBaqA1iDXVIZtCwsLqz2f1/voyF+3xx13HHr16qVmxElB/jPPPMPrHWYybCKTRk455RRYrVa1SVApExzkWDIKvN6RJVmc448/Hr/88ktEv74Z7EToB5X8kJo/f361IQC5Lalpipz27durb4rga19cXKxmZfHah05qwCXQkWGUL774Ql3fYPJ1brPZql1vmZouY/C83uEjPz+cTievd5gNGjRIDRlKFs3YevfurepIjGNe78gqLS3Fr7/+qpYKiejX91GVN1OdZs6cqWYAvfbaa9oPP/ygjR49WsvJydEKCgpifWoJMXPim2++UZt8CU+ePFkdb926VT3+6KOPqmv93//+V/vuu++0Sy65RGvfvr1WUVER61M/5tx8881adna29uWXX2q7du0KbOXl5YHn3HTTTVqbNm20L774Qlu1apXWv39/tVHD3H333Wq22+bNm9XXr9w2mUzaZ599ph7n9Y6s4NlYgtc7vP7yl7+onyfy9f31119rgwcP1vLy8tRMz0hebwY7EfTcc8+pT5rdbldT0ZctWxbrU0oICxYsUEHOodvIkSMD08/vu+8+LT8/XwWcgwYN0jZu3Bjr0z4m1XadZXv11VcDz5Eg8pZbblHTo9PS0rRLL71UBUTUMDfccIPWtm1b9XOjSZMm6uvXCHQEr3d0gx1e7/C68sortebNm6uv75YtW6rbv/zyS8Svt0n+d/SJKCIiIqL4xJodIiIiSmgMdoiIiCihMdghIiKihMZgh4iIiBIagx0iIiJKaAx2iIiIKKEx2CEiIqKExmCHiIiIEhqDHSI6ZmzZsgUmk0n1LYqU66+/HsOGDYvY6xNR9DHYIaKokUBCgpVDt3PPPbdeH9+6dWvs2rUL3bp1i/i5ElHisMb6BIgouUhg8+qrr1a7z+Fw1OtjLRaL6mpPRBQKZnaIKKoksJGAJXhr1KiRekyyPFOmTMF5552H1NRUdOjQAe+8806dw1gHDx7EiBEj0KRJE/X8Tp06VQuk1q1bh7PPPls9lpubi9GjR6O0tDTwuNfrxdixY5GTk6Mev+uuu6Q5crXz9fl8mDhxItq3b69ep0ePHtXO6UjnQESxx2CHiOLKfffdh+HDh+Pbb79VQcQf/vAHbNiwoc7n/vDDD/j000/VcyRQysvLU4+VlZVh6NChKpBauXIlZs+ejc8//xy33XZb4OOffPJJvPbaa3jllVfw1Vdf4cCBA3j//fervYcEOq+//jqmTp2K9evX484778Q111yDhQsXHvEciChOHHXfdCKieho5cqRmsVi09PT0atvDDz+sHpcfSTfddFO1j+nXr5928803q+PNmzer53zzzTfq9kUXXaT98Y9/rPW9pk2bpjVq1EgrLS0N3Pfxxx9rZrNZKygoULebN2+uTZo0KfC42+3WWrVqpV1yySXqdmVlpZaWlqYtWbKk2muPGjVKu+qqq454DkQUH1izQ0RR9bvf/U5lP4I1btw4cNy/f/9qj8ntumZf3XzzzSoLtGbNGgwZMkTNohowYIB6TLIsMuSUnp4eeP7AgQPVsNTGjRuRkpKiip379esXeNxqtaJ3796BoaxffvkF5eXlOOecc6q9r8vlwsknn3zEcyCi+MBgh4iiSoKP4447LiyvJbU9W7duxSeffIJ58+Zh0KBBuPXWW/HEE0+E5fWN+p6PP/4YLVu2rLWoOtLnQERHjzU7RBRXli1bVuP2iSeeWOfzpTB45MiR+M9//oOnn34a06ZNU/fLx0jdj9TuGL7++muYzWaccMIJyM7ORvPmzbF8+fLA4x6PB6tXrw7c7tKliwpqtm3bpgK04E2mwR/pHIgoPjCzQ0RR5XQ6UVBQUO0+GT4yinqlkFiGkk477TS88cYbWLFiBV5++eVaX2vcuHHo1asXunbtql53zpw5gcBIipvHjx+vgpD7778fe/fuxe23345rr70W+fn56jljxozBo48+qmZQde7cGZMnT0ZhYWHg9TMzM/HXv/5VFSXL8JecU1FRkQqasrKy1Gsf7hyIKD4w2CGiqJo7d67KqASTTMuPP/6ojh944AHMnDkTt9xyi3reW2+9pTIstbHb7bjnnnvUlHSZ9n366aerjxVpaWn43//+pwKaPn36qNtSWyMBjeEvf/mLqtuRoEUyPjfccAMuvfRSFdAYJkyYoDI3Mitr06ZNapr6Kaecgn/84x9HPAciig8mqVKO9UkQEQlZQ0emfrNdAxGFE2t2iIiIKKEx2CEiIqKExpodIoobHFUnokhgZoeIiIgSGoMdIiIiSmgMdoiIiCihMdghIiKihMZgh4iIiBIagx0iIiJKaAx2iIiIKKEx2CEiIiIksv8H9tYoC+SV/Q4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def run_episodes(\n", + " env: DynaMazeEnv,\n", + " planning_steps: int,\n", + " episodes: int = 50,\n", + " seed: int = 0,\n", + ") -> np.ndarray:\n", + " \"\"\"Run one experiment of Dyna-Q on a given environment, for a fixed number of episodes and planning steps.\n", + "\n", + " Args:\n", + " env (DynaMazeEnv): the maze environment\n", + " planning_steps (int): the Dyna parameter n = number of model/planning updates per real step.\n", + " episodes (int, optional): number of episodes to run. Defaults to 50.\n", + " seed (int, optional): random seed for reproducibility. Defaults to 0.\n", + "\n", + " \"\"\"\n", + " rng = np.random.default_rng(\n", + " seed,\n", + " ) # create a random number generator for reproducibility\n", + " agent = DynaQAgent(\n", + " env.n_states, env.n_actions, planning_steps=planning_steps, rng=rng,\n", + " ) # Initializes a new Dyna-Q agent with:\n", + " # Q-table size (n_states, n_actions)\n", + " # planning_steps = the Dyna parameter n = number of model/planning updates per real step.\n", + " # rng for reproducibility\n", + "\n", + " steps_per_episode = [] # This will store one integer per episode: the number of steps until reaching goal.\n", + "\n", + " for _ in range(episodes): # Loop episodes\n", + " s = env.reset() # Reset the environment at the start for each episode\n", + " done = False # done is the terminal indicator from the environment, it is false at the beginning\n", + " steps = 0 # steps counts how many environment interactions happen this episode.\n", + " while not done: # interaction until terminal\n", + " a = agent.act(s) # Choose action using epsilon-greedy policy\n", + " sp, r, done = env.step(\n", + " a,\n", + " ) # this is the environment transition: sp = next state s'; r = reward ; done = whether goal reached.\n", + " agent.observe(\n", + " s, a, r, sp,\n", + " ) # This is the whole Dyna update : the agent will learn from this real transition + plan from model\n", + " s = sp # this line advances state variable, now sp becomes the current state\n", + " steps += 1 # add + 1 into the step counter\n", + " steps_per_episode.append(steps) # Store episode length after reaching the goal\n", + "\n", + " return np.array(steps_per_episode) # return the number of step as NumPy array\n", + "\n", + "\n", + "def replicate_figure() -> None:\n", + " \"\"\"Run multiple independent experiments and plot averages for different planning steps n.\"\"\"\n", + " env = DynaMazeEnv() # call the environment\n", + " ns = [\n", + " 0,\n", + " 5,\n", + " 50,\n", + " ] # These are different values of Dyna-Q planning step where 0 = pure Q-learning (no planning)\n", + " runs = 30 # runs: number of independent experimental runs (different seeds)\n", + " episodes = 50 # episodes: number of episodes per run\n", + " results = {n: [] for n in ns} # Storage dictionary, which creates a dictionary like\n", + "\n", + " for rep in range(runs): # Run experiments\n", + " seed = rep # TO DO # choose seeds, what is your stratgy to choose different seed ? Hint. for each rep, we need to choose different seed.\n", + " for (\n", + " n\n", + " ) in ns: # Inner loop: compare different planning values using the same seed\n", + " env = DynaMazeEnv() # call a new environment for this condition\n", + " results[n].append(\n", + " run_episodes(env, n, episodes, seed),\n", + " ) # run_episodes creates a new agent and runs 50 episodes, returns an array of length 50\n", + " # Append that array into results[n]\n", + "\n", + " avg = {\n", + " n: np.mean(results[n], axis=0) for n in ns\n", + " } # results[n] is a list of arrays ; axis=0 means average over runs, for each episode index.\n", + "\n", + " for n in ns: # Plots one curve per n with X-axis = episode number (0, ..., 49) and Y-axis = average steps to reach goal\n", + " plt.plot(avg[n], label=f\"n={n}\")\n", + "\n", + " plt.xlabel(\"Episodes\")\n", + " plt.ylabel(\"Steps per episode\")\n", + " plt.legend()\n", + " plt.show()\n", + "\n", + "\n", + "replicate_figure()\n" + ] + }, + { + "cell_type": "markdown", + "id": "22d14dd3", + "metadata": {}, + "source": [ + "**Exercise 4.** How and why does increasing the planning step `n` speed up learning?\n" + ] + }, + { + "cell_type": "markdown", + "id": "e763d124-f973-4c90-b23f-986cef3b7426", + "metadata": {}, + "source": [ + "More planning steps -> more simulated Q-updates per real step -> faster\n", + "\n", + "value propagation -> improved sample efficiency (fewer real interactions)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "studies", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}