n = 1000;
N = 4;
%rho = 2.0;
rho = 0.0;
var = 0.5;

arms = 1:1:N;
mu = [0.5 0.8 0.3 0.79]';
hmu = zeros(N, 1);
cumulX = zeros(N, 1);
pulls = zeros(N, 1);
regret = zeros(n, 1);

% Create figure
figure1 = figure('XVisual','');

for t=1:length(mu)
    best_arm = t;
    %sample = (rand < mu(best_arm));
    sample = mu(best_arm)+var*randn(1,1);
    
    cumulX(best_arm) = cumulX(best_arm) + sample;
    pulls(best_arm) = pulls(best_arm)+1;
    hmu(best_arm) = cumulX(best_arm)/pulls(best_arm);
    regret(t) = pulls'*(max(mu)-mu);
end    
conf = rho*sqrt(log(t)./(2.*pulls));
plot_arms(figure1, arms, hmu, conf, mu, pulls, best_arm, regret, t);
waitforbuttonpress;

for t=length(mu):n
    conf = rho*sqrt(log(t)./(2.*pulls));
    B = hmu + conf;
    [best_value, best_arm] = max(B);
    
    plot_arms(figure1, arms, hmu, conf, mu, pulls, best_arm, regret, t)
    
    %sample = (rand < mu(best_arm));
    sample = mu(best_arm)+var*randn(1,1);
    
    cumulX(best_arm) = cumulX(best_arm) + sample;
    pulls(best_arm) = pulls(best_arm)+1;
    hmu(best_arm) = cumulX(best_arm)/pulls(best_arm);
    regret(t) = pulls'*(max(mu)-mu);
    %pause(0.2);
    
   if (mod(t, 10) == 0)
       waitforbuttonpress;
       disp(['Cumulative regret = ' num2str(regret(t))]);
      %pause(10);
   end
end



