How can I structure two DQN models that take state and action as inputs and output Q$-$values for those state$-$action pairs? I have a skeleton of the code here : import numpy as np

import pandas as pd

import gymnasium as gym

def load$\_$offline$\_$data$($path$,$ min$\_$score$)$:

state$\_$data $=[]$

action$\_$data $=[]$

reward$\_$data $=[]$

next$\_$state$\_$data $=[]$

terminated$\_$data $=[]$

dataset $=$ pd$.$read$\_$csv$($path$)$

dataset$\_$group $=$ dataset.groupby$(\text{'}$Play #$\text{'})$

for play$\_$no$,$ df in dataset$\_$group:

state $=$ np$.$array$($df$.$iloc$[$:$,1])$

state $=$ np$.$array$([$np$.$fromstring$($row$[1$:$-1],$ dtype$=$np$.$float$32,$ sep$=\text{'}\text{'})$ for row in state$])$

action $=$ np$.$array$($df$.$iloc$[$:$,2]).$astype$($int$)$

reward $=$ np$.$array$($df$.$iloc$[$:$,3]).$astype$($np$.$float$32)$

next$\_$state $=$ np$.$array$($df$.$iloc$[$:$,4])$

next$\_$state $=$ np$.$array$([$np$.$fromstring$($row$[1$:$-1],$ dtype$=$np$.$float$32,$ sep$=\text{'}\text{'})$ for row in next$\_$state$])$

terminated $=$ np$.$array$($df$.$iloc$[$:$,5]).$astype$($int$)$

total$\_$reward $=$ np$.$sum$($reward$)$

if total$\_$reward$>=$min$\_$score:

state$\_$data.append$($state$)$

action$\_$data.append$($action$)$

reward$\_$data.append$($reward$)$

next$\_$state$\_$data.append$($next$\_$state$)$

terminated$\_$data.append$($terminated$)$

state$\_$data $=$ np$.$concatenate$($state$\_$data$)$

action$\_$data $=$ np$.$concatenate$($action$\_$data$)$

reward$\_$data $=$ np$.$concatenate$($reward$\_$data$)$

next$\_$state$\_$data $=$ np$.$concatenate$($next$\_$state$\_$data$)$

terminated$\_$data $=$ np$.$concatenate$($terminated$\_$data$)$

return state$\_$data, action$\_$data, reward$\_$data, next$\_$state$\_$data, terminated$\_$data

def plot$\_$reward$($total$\_$reward$\_$per$\_$episode, window$\_$length$)$:

# This function should display:

# $($i$)$ total reward per episode.

# $($ii$)$ moving average of the total reward. The window for moving average

# should slide by one episode every time.

pass

def DQN$\_$training$($env$,$ offline$\_$data, use$\_$offline$\_$data$)$:

# The function should return the final trained DQN model and total reward

# of every episode.

pass

# Initiate the lunar lander environment.

# NO RENDERING. It will slow the training process.

env $=$ gym.make$(\text{'}$LunarLander$-$v$2\text{'})$

# Load the offline data collected in step $3.$ Also, process the dataset.

path $=$ 'lunar$\_$dataset.csv$\text{'}$ # This should contain the path to the collected dataset.

min$\_$score $=-$np$.$Inf # The minimum total reward of an episode that should be used for training.

offline$\_$data $=$ load$\_$offline$\_$data$($path$,$ min$\_$score$)$

# Train DQN model of Architecture type $1$

use$\_$offline$\_$data $=$ True # If True then the offline data will be used. Else, offline data will not be used.

final$\_$model, total$\_$reward$\_$per$\_$episode $=$ DQN$\_$training$($env$,$ offline$\_$data, use$\_$offline$\_$data$)$

# Save the final model

final$\_$model.save$(\text{'}$lunar$\_$lander$\_$model.h$5\text{'})$ # This line is for Keras. Replace this appropriate code.

# Plot reward per episode and moving average reward

window$\_$length $=50$ # Window length for moving average reward.

plot$\_$reward$($total$\_$reward$\_$per$\_$episode, window$\_$length$)$

env.close$()$