Please fill the parts with "@":

clearvars clc addpath('../Generation') addpath('../Basic_blocks') addpath('../Algorithms')

% Loading scenarios % =========================== scenario=2; [data_reg set_up]=scenarios_regression(scenario);

% Definition of the problem %=================================== loss_LS_L2 = @(N,A,x,y,lambda) (1/N*(A*x-y)'*(A*x-y)+lambda/2*x'*x); grad_LS_L2 = @(N,A,x,y,lambda) (2/N*A'*(A*x-y)+lambda*x); Hess_LS_L2 = @(N,A,x,y,lambda) (2/N*A'*A+lambda*eye(length(x)));

% Definition of the problem %=================================== loss_LS_L2 = @(N,A,x,y,lambda) (1/N*(A*x-y)'*(A*x-y)+lambda/2*x'*x); grad_LS_L2 = @(N,A,x,y,lambda) (2/N*A'*(A*x-y)+lambda*x); Hess_LS_L2 = @(N,A,x,y,lambda) (2/N*A'*A+lambda*eye(length(x)));

% Different ways to solve theoreticaly the LS %========================================= % Solution of the empirical risk using CVX x_L2_cvx=solver_cvx(set_up,@(N,A,x,y,lambda) loss_LS_L2(N,A,x,y,lambda)); x_opt=inv(2/set_up.Niter_train*set_up.Utrain(:,1:set_up.M+1)'*set_up.Utrain(:,1:set_up.M+1)+set_up.Lambda*eye(set_up.M+1))... *2/set_up.Niter_train*set_up.Utrain(:,1:set_up.M+1)'*set_up.ytrain(:,1); loss_opt=loss_LS_L2(set_up.Niter_train,set_up.Utrain(:,1:set_up.M+1),x_L2_cvx,set_up.ytrain(:,1),set_up.Lambda); [x_L2_cvx x_opt ]

% Gradient descent out_gd =grad_FOM(set_up,@(N,A,x,y,lambda) grad_LS_L2(N,A,x,y,lambda)); loss_grad=eval_loss(out_gd,set_up,@(N,A,x,y,lambda) loss_LS_L2(N,A,x,y,lambda));

% Newton algorithm out_hess =grad_SOM(set_up,@(N,A,x,y,lambda) grad_LS_L2(N,A,x,y,lambda),@(N,A,x,y,lambda) Hess_LS_L2(N,A,x,y,lambda)); loss_hess=eval_loss(out_hess,set_up,@(N,A,x,y,lambda) loss_LS_L2(N,A,x,y,lambda));

% BFGS algorithm out_BFGS =BFGS(set_up,@(N,A,x,y,lambda) grad_LS_L2(N,A,x,y,lambda)); loss_BFGS=eval_loss(out_BFGS,set_up,@(N,A,x,y,lambda) loss_LS_L2(N,A,x,y,lambda)); hold off

figure(1) % Plot of learning curves plot(1:set_up.Niter_train,10*log10(sum((loss_grad-loss_opt*ones(1,set_up.Niter_train)).^2,1)),'b','LineWidth',3), hold plot(1:set_up.Niter_train,10*log10(sum((loss_hess-loss_opt*ones(1,set_up.Niter_train)).^2,1)),'r','LineWidth',3), plot(1:set_up.Niter_train,10*log10(sum((loss_BFGS-loss_opt*ones(1,set_up.Niter_train)).^2,1)),'g','LineWidth',3), hold off grid xlabel('Iterations') ylabel('MSE') title('Ridge Algorithm') legend('Gradient','Newton','BFGS') figure(2)

% Let's make a zoom % Plot of learning curves show=30; plot(1:show,10*log10(sum((loss_grad(1,show)-loss_opt*ones(1,show)).^2,1)),'b','LineWidth',3), hold plot(1:show,10*log10(sum((loss_hess(1:show)-loss_opt*ones(1,show)).^2,1)),'r','LineWidth',3), plot(1:show,10*log10(sum((loss_BFGS(1:show)-loss_opt*ones(1,show)).^2,1)),'g','LineWidth',3), hold off grid xlabel('Iterations') ylabel('MSE') title('Ridge Algorithm (Zoom)') legend('Gradient','Newton','BFGS')