What is the learned Q

table for the following code? Please run the code and show the output.

import numpy as np

import matplotlib.pyplot as plt

# Grid world size

WORLD$\_$SIZE $=10$

# Percentage of cells occupied by obstacles

OBSTACLE$\_$DENSITY $=0\mathrm{.}15$

# Learning parameters

ALPHA $=0\mathrm{.}5$

GAMMA $=0\mathrm{.}9$

EPSILON $=0\mathrm{.}1$

def initialize$\_$world$()$:

# Create empty grid

world $=$ np$.$zeros$(($WORLD$\_$SIZE, WORLD$\_$SIZE$))$

# Set start in bottom left

world$[0,0]=2$

# Set goal in top right

world$[-1,-1]=3$

# Add random obstacles

num$\_$obstacles $=$ int$($OBSTACLE$\_$DENSITY $*$ WORLD$\_$SIZE$**2)$

obstacle$\_$indices $=$ np$.$random.choice$($range$($WORLD$\_$SIZE$**2),$ size$=$num$\_$obstacles, replace$=$False$)$

for i in obstacle$\_$indices:

x $=$ i $//$ WORLD$\_$SIZE

y $=$ i $\%$ WORLD$\_$SIZE

world$[$x$,$y$]=1$

return world

def initialize$\_$q$\_$values$()$:

# Q$($s$,$a$)$ initialized to $0$ for all s$,$a

q$\_$values $=\{\}$

for x in range$($WORLD$\_$SIZE$)$:

for y in range$($WORLD$\_$SIZE$)$:

for a in range$(4)$: # up$,$ down, left, right

q$\_$values$[($x$,$y$,$a$)]=0\mathrm{.}0$

return q$\_$values

def epsilon$\_$greedy$($state$,$ q$\_$values, epsilon$)$:

# With probability epsilon, take random action

# Otherwise, take greedy action based on current Q values

if np$.$random.rand$()<$ epsilon:

action $=$ np$.$random.randint$(4)$

else:

values $=[$q$\_$values$[($state$[0],$ state$[1],$ a$)]$ for a in range$(4)]$

action $=$ np$.$argmax$($values$)$

return action

def update$\_$q$\_$value$($state$,$ action, reward, next$\_$state, q$\_$values, alpha, gamma$)$:

# Q$-$learning update rule

max$\_$q$\_$next $=$ max$([$q$\_$values$[($next$\_$state$[0],$ next$\_$state$[1],$ a$)]$ for a in range$(4)])$

q$\_$values$[($state$[0],$ state$[1],$ action$)]+=$ alpha $*($reward $+$ gamma $*$ max$\_$q$\_$next $-$ q$\_$values$[($state$[0],$ state$[1],$ action$)])$

return q$\_$values

def check$\_$goal$($state$)$:

return state $==($WORLD$\_$SIZE$-1,$ WORLD$\_$SIZE$-1)$

if $\_\_$name$\_\_=="\_\_$main$\_\_"$:

# Create world

world $=$ initialize$\_$world$()$

# Initialize Q values

q$\_$values $=$ initialize$\_$q$\_$values$()$

# Track metrics

steps$\_$per$\_$episode $=[]$

sse $=[]$

for episode in range$(1000)$:

# Reset agent to start position

state $=(0,0)$

step $=0$

episode$\_$sse $=0$

while not check$\_$goal$($state$)$:

# Choose action using epsilon$-$greedy

action $=$ epsilon$\_$greedy$($state$,$ q$\_$values, EPSILON$)$

# Take action and get reward$/$next state

if action $==0$: # up

next$\_$state $=($state$[0]-1,$ state$[1])$

elif action $==1$: # down

next$\_$state $=($state$[0]+1,$ state$[1])$

elif action $==2$: # left

next$\_$state $=($state$[0],$ state$[1]-1)$

else: # right

next$\_$state $=($state$[0],$ state$[1]+1)$

reward $=-0\mathrm{.}1$

if world$[$next$\_$state$]==1$: # Hit obstacle

reward $=-1$

next$\_$state $=$ state # Stay in current state

if check$\_$goal$($next$\_$state$)$:

reward $=10$

# Update Q value

q$\_$values $=$ update$\_$q$\_$value$($state$,$ action, reward, next$\_$state, q$\_$values, ALPHA, GAMMA$)$

# Calculate SSE

episode$\_$sse $+=($reward $+$ GAMMA $*$ max$([$q$\_$values$[($next$\_$state$[0],$ next$\_$state$[1],$ a$)]$ for a in range$(4)])-$ q$\_$values$[($state$[0],$ state$[1],$ action$)])**2$

# Update state

state $=$ next$\_$state

step $+=1$

steps$\_$per$\_$episode.append$($step$)$

sse.append$($episode$\_$sse$)$

# Plot results

plt$.$plot$($steps$\_$per$\_$episode$)$

plt$.$xlabel$(\text{'}$Episode$\text{'})$

plt$.$ylabel$(\text{'}$Steps per episode'$)$

plt$.$savefig$(\text{'}$steps$.$png$\text{'})$

plt$.$plot$($sse$)$

plt$.$xlabel$(\text{'}$Episode$\text{'})$

plt$.$ylabel$(\text{'}$Sum squared error'$)$

plt$.$savefig$(\text{'}$sse$.$png$\text{'})$

# Print learned policy

policy $=\{\}$

for x in range$($WORLD$\_$SIZE$)$:

for y in range$($WORLD$\_$SIZE$)$:

values $=[$q$\_$values$[($x$,$y$,$a$)]$ for a in range$(4)]$

policy$[($x$,$y$)]=$ np$.$argmax$($values$)$

print$("$Learned Optimal Policy:"$)$

print$($policy$)$

# Print the learned Q$-$table

print$("$Learned Q$-$table:"$)$

print$($q$\_$table$)$