The goal of this assignment is to implement QLearning method on Taxi$-$v$3$ enviroment at openai gym framework.

Your task in this enviroment is to pick up the passenger at one location and drop him off in another, located at possible $4$ locations $($labeled by different letters$).$ In the example given below, you are expected to pick him up at Y and drop him at G$.$ You receive $+20$ points for a successful dropoff, and lose $1$ point for every timestep it takes. There is also a $10$ point penalty for illegal pick$-$up and drop$-$off actions.

Note that dynamics of the model are assumed to be unknown.

below is the original code, impliment the QLearning method accordingly:

import gymnasium as gym

import time

import numpy as np

import os

import random

def qLearning$($env$)$:

nS $=$ env.observation$\_$space.n

nA $=$ env.action$\_$space.n

Q $=$ np$.$zeros $([$nS$,$ nA$],$ dtype$=$np$.$int$32)$

alpha $=0\mathrm{.}8$

gamma $=0\mathrm{.}9$

epsilon $=1$

num$\_$iter $=10000$

for i in range $($num$\_$iter$)$:

s$,$ actions $=$ env. reset$()$

for step in range $(100)$:

action $=$ env.action$\_$space.sample$()$

#action $=$ np$.$argmax$($Q$[$s$])$

sp$,$ reward, done, info $=$ env.step $($action$)$

Q$[$s$,$ action$]=$ Q$[$s$,$action$]+$ alpha $*($reward $+$gamma $*$np$.$max $($Q$[$sp$,$:$])-$ Q$[$s$,$ action$])$

S $=$ sp

if i$\%1000==0$ :

print $($f$"$Episode $\{$i$\}")$

return Q

def SARSA $($env$)$:

nS $=$ env.observation$\_$space.n

nA $=$ env.action$\_$space.n

Q $=$ np$.$zeros $([$nS$,$nA$],$ dtype$=$np$.$ int$32)$

alpha $=0\mathrm{.}8$

gamma $=0\mathrm{.}9$

epsilon $=1$

num$\_$iter $=1000$

for i in range $($num$\_$iter$)$:

S$,$ actions $=$ env.reset$()$

a $=$ env.action$\_$space.sample$()$

for step in range $(100)$:

sp$,$ reward, done, truncated, info $=$ env. step$($a$)$

ap $=$ np$.$argmax $($Q$[$sp$])$

Q$[$S$,$ a$]=$ Q$[$S$,$a$]+$ alpha $*($reward $+$ gamma $*$ Q$[$sp$,$ ap$]-$Q$[$S$,$a$])$

S $=$ sp

a $=$ ap

if i$\%1000==0$ :

print$($f$"$Episode $\{$i$\}")$

return Q

env $=$ gym.make$(\text{'}$Taxi$-$v$3\text{'},$ render$\_$mode$=$"human" $)$

observation,info $=$ env.reset $()$

Q $=$ SARSA $($env$)$

observation $=$ env. reset$()$

done$=$False

sumreward $=0$

while not done:

os$.$ system$(\text{'}$cls$\text{'})$

env. render $()$

action $=$ np$.$argmax $($Q$[$observation$])$

observation, reward, done, truncated, info $=$ env. step$($action$)$

sumreward $+=$ reward

time.sleep $(0\mathrm{.}5)$

if done:

observation $=$ env. reset $()$

print $(\text{'}$done with reward:$\text{'},$ reward$)$

env. close$()$