using LinearAlgebra, Zygote, ForwardDiff, Printf

using CSV$,$ DataFrames

using StatsBase: mean

using Parameters

using Distributions

using Random

using BSON

using ProgressMeter

using ProgressBars

### Below you will implement your linear regression object from scratch.

###

### you will also imply some penalty methods as well.

#### $-----$ ###

#### Before getting started you should write your student$\_$number in integer format

const student$\_$number::Int$64=0$ ## $<---$replace $0$ by your student$\_$number

### $----$ ###

### This hw is a bit harder than the usual ones, so you may want to

### take a look at $--$Julia official website

### Before you get started, $---$ you gotta look at do end syntax of Julia

### Instead of mean square loss we shall use Huber loss which

### sometimes performs better when your dataset contains some out of distribution points

### We split it into two parts the first one is for scalars the second one is for vectors

### remember that you will multiple dispatch!!!

function huber$\_$loss$($y$\_$pred::T$,$ y$\_$true::T; $\backslash$delta ::Float$64=0\mathrm{.}1)$ where T $<$: Real

loss $=$ abs$($y$\_$pred $-$ y$\_$true$)$

if loss $<=\backslash$delta

return $0\mathrm{.}5*$ loss$^2$

else

return $\backslash$delta $*($loss $-0\mathrm{.}5*\backslash$delta $)$

end

end

function huber$\_$loss$($y$\_$pred::AbstractArray$\{$T$\},$ y$\_$true::AbstractArray$\{$T$\})$ where T $<$: Real

n $=$ length$($y$\_$pred$)$

total$\_$loss $=0\mathrm{.}0$

for i in $1$:n

total$\_$loss $+=$ huber$\_$loss$($y$\_$pred$[$i$],$ y$\_$true$[$i$])$

end

return total$\_$loss $/$ n

end

function unit$\_$test$\_$huber$\_$loss$()$::Bool

Random.seed!$(0)$

@assert huber$\_$loss$(1\mathrm{.}0,1\mathrm{.}0)==0\mathrm{.}0$

@assert huber$\_$loss$(1\mathrm{.}0,2\mathrm{.}0$; $\backslash$delta $=0\mathrm{.}9)==0\mathrm{.}49500000000000005$

@assert isapprox$($huber$\_$loss$($randn$(100,100),$randn$(100,100)),0\mathrm{.}10791842,$ atol $=1$e$-2)$

@assert isapprox$($huber$\_$loss$($randn$(100),$randn$(100)),0\mathrm{.}107945,$ atol $=1$e$-2)$

@info "You can not stop now comrade!!! jump to the next exercise!!!"

return $1$

end

## See you have implemented huber$\_$loss$()$ well??

unit$\_$test$\_$huber$\_$loss$()$

### create a roof for the logistic regression LogisticClassifier

abstract type LinearRegression end

mutable struct linear$\_$regression $<$: LinearRegression

## This part is given to you!!!

## Realize that we have fields: $\backslash$theta and bias.

$\backslash$theta ::AbstractVector

bias::Real

linear$\_$regression$($n::Int$64)=$ new$(0\mathrm{.}004*$randn$($n$),$ zero$(1))$

end

### write the forwardpass function

function $($lr::linear$\_$regression$)($X::Matrix$\{$T$\})$ where T$<$:Real

## This dude is the forward pass function!!!

return X $*$ lr$.\backslash$theta $.+$ lr$.$bias

end

function unit$\_$test$\_$forward$\_$pass$()$::Bool

try

linear$\_$regression$(20)($randn$(10,20))$

catch ERROR

error$("$SEG$-$FAULT!!!!"$)$

end

@info "Real test started!!!!"

for i in ProgressBar$(1$:$10000)$

sleep$(0\mathrm{.}0001)$

lr $=$ linear$\_$regression$(3)$

x $=$ randn$(2,3)$

@assert lr$($x$)==$ x$*$lr$.\backslash$theta $.+$ lr$.$bias

end

@info "Oki doki!!!"

return $1$

end

### Let's give a try!!!!!!

unit$\_$test$\_$forward$\_$pass$()$

## we shall now implement fit! method!!!

## before we get ready run the next $5$ lines to see in this setting grad function returns a dictionary:

#$=$

lr $=$ linear$\_$regression$(10)$

val, grad $=$ Zygote.withgradient$($lr$)$ do lr

norm$($lr$.\backslash$theta $)^2+$ lr$.$bias$^2$

end

$=$#

function update$\_$grads!$($lr::LinearRegression, learning$\_$rate::Float$64,$ grads$)$

## Here you will implement update$\_$grads, this function returns nothing.

## Search for setfield! and getfield functions.

## x $-=$ learning$\_$rate$*$grads will happen here!!!

lr$.\backslash$theta $.-=$ learning$\_$rate $*$ grads$[$:$\backslash$theta $]$

lr$.$bias $-=$ learning$\_$rate $*$ grads$[$:bias$]$

return nothing

end

function fit!$($lr::linear$\_$regression,

X::AbstractMatrix,

y::AbstractVector;

learning$\_$rate::Float$64=0\mathrm{.}00001,$

max$\_$iter::Integer $=5,$

$\backslash$lambda ::Float$64=0\mathrm{.}01)$

##

for i in $1$:max$\_$iter

val, grad $=$ Zygote.withgradient$($lr$)$ do lr

norm$($lr$.\backslash$theta $)^2+$ lr$.$bias$^2$

end

end

return nothing

end

## Let's give a try!!!

lr $=$ linear$\_$regression$(20)$

X $=$ randn$(100,20)$

y $=$ randn$(100)$

fit!$($lr$,$ X$,$ y; learning$\_$rate $=0\mathrm{.}00001,$ max$\_$iter $=10000)$

### Things seem to work fine!!!

function unit$\_$test$\_$for$\_$fit$()$

Random.seed!$(0)$

lr $=$ linear$\_$regression$(20)$

X $=$ randn$(100,20)$

y $=$ randn$(100)$

fit!$($lr$,$ X$,$ y; learning$\_$rate $=0\mathrm{.}0001,$ max$\_$iter $=10000,\backslash$lambda $=0\mathrm{.}1)$

@assert norm$($lr$.\backslash$theta $)^2+$ lr$.$bias$^2<0\mathrm{.}01$ "Your penalty method does not work!!!"

@assert mean$(($lr$($X$)-$ y$).^2)<1\mathrm{.}3"$Yo do not fit perfectly!!!!"

@info "Okito dokito buddy!!!"

return $1$

end

## Run next line to see what happens??? ##

unit$\_$test$\_$for$\_$fit$()$

i can't change test functions