$1$

$2$

$3$

$4$

$5$

$6$

$7$

$8$

$9$

$10$

$11$

$12$

$13$

$14$

$15$

$16$

$17$

$18$

$19$

$20$

$21$

$22$

$23$

$24$

$25$

$26$

$27$

$28$

$29$

$30$

$31$

$32$

$33$

$34$

$35$

$36$

$37$

$38$

$39$

$40$

$41$

$42$

$43$

$44$

$45$

$46$

$47$

$48$

$49$

$50$

$51$

$52$

$53$

$54$

$55$

$56$

$57$

$58$

$59$

$60$

$61$

$62$

$63$

$64$

$65$

$66$

$67$

$68$

$69$

$70$

$71$

$72$

$73$

$74$

import numpy as np

import pandas as pd

import gymnasium as gym

def load$\_$offline$\_$data$($path$,$ min$\_$score$)$:

state$\_$data $=[]$

action$\_$data $=[]$

reward$\_$data $=[]$

next$\_$state$\_$data $=[]$

terminated$\_$data $=[]$

dataset $=$ pd$.$read$\_$csv$($path$)$

dataset$\_$group $=$ dataset.groupby$(\text{'}$Play #$\text{'})$

for play$\_$no$,$ df in dataset$\_$group:

state $=$ np$.$array$($df$.$iloc$[$:$,1])$

state $=$ np$.$array$([$np$.$fromstring$($row$[1$:$-1],$ dtype$=$np$.$float$32,$ sep$=\text{'}\text{'})$ for row in state$])$

action $=$ np$.$array$($df$.$iloc$[$:$,2]).$astype$($int$)$

reward $=$ np$.$array$($df$.$iloc$[$:$,3]).$astype$($np$.$float$32)$

next$\_$state $=$ np$.$array$($df$.$iloc$[$:$,4])$

next$\_$state $=$ np$.$array$([$np$.$fromstring$($row$[1$:$-1],$ dtype$=$np$.$float$32,$ sep$=\text{'}\text{'})$ for row in next$\_$state$])$

terminated $=$ np$.$array$($df$.$iloc$[$:$,5]).$astype$($int$)$

total$\_$reward $=$ np$.$sum$($reward$)$

if total$\_$reward$>=$min$\_$score:

state$\_$data.append$($state$)$

action$\_$data.append$($action$)$

reward$\_$data.append$($reward$)$

next$\_$state$\_$data.append$($next$\_$state$)$

terminated$\_$data.append$($terminated$)$

state$\_$data $=$ np$.$concatenate$($state$\_$data$)$

action$\_$data $=$ np$.$concatenate$($action$\_$data$)$

reward$\_$data $=$ np$.$concatenate$($reward$\_$data$)$

next$\_$state$\_$data $=$ np$.$concatenate$($next$\_$state$\_$data$)$

terminated$\_$data $=$ np$.$concatenate$($terminated$\_$data$)$

rturn state$\_$data, action$\_$data, reward$\_$data, next$\_$state$\_$data, terminated$\_$data

def plot$\_$reward$($total$\_$reward$\_$per$\_$episode, window$\_$length$)$:

# This function should display:

# $($i$)$ total reward per episode.

# $($ii$)$ moving average of the total reward. The window for moving average

# should slide by one episode every time.

pass

def DQN$\_$training$($env$,$ offline$\_$data, use$\_$offline$\_$data$)$:

# The function should return the final trained DQN model and total reward

# of every episode.

pass

# Initiate the lunar lander environment.

# NO RENDERING. It will slow the training process.

env $=$ gym.make$(\text{'}$LunarLander$-$v$2\text{'})$

# Load the offline data collected in step $3.$ Also, process the dataset.

path $=$ 'lunar$\_$dataset.csv$\text{'}$ # This should contain the path to the collected dataset.

min$\_$score $=-$np$.$Inf # The minimum total reward of an episode that should be used for training.

offline$\_$data $=$ load$\_$offline$\_$data$($path$,$ min$\_$score$)$

# Train DQN model of Architecture type $1$

use$\_$offline$\_$data $=$ True # If True then the offline data will be used. Else, offline data will not be used.

final$\_$model, total$\_$reward$\_$per$\_$episode $=$ DQN$\_$training$($env$,$ offline$\_$data, use$\_$offline$\_$data$)$

# Save the final model

final$\_$model.save$(\text{'}$lunar$\_$lander$\_$model.h$5\text{'})$ # This line is for Keras. Replace this appropriate code.

# Plot reward per episode and moving average reward

window$\_$length $=50$ # Window length for moving average reward.

plot$\_$reward$($total$\_$reward$\_$per$\_$episode, window$\_$length$)$

env.close$()$

this is the skeleton of training.py

give me the code for dqn architecute $1$ without using offline data. Which means take action or data from env it self

Make sure when u use the traninn.py code above give dqn architecuture which takes state and action as input and outputs only one q value with respect to that action. Also make sure you get a increasing reward trend when u plot it$.$