W1=[-0.27
   -0.41];
b1=[-0.48
   -0.13];
W2=[0.09 -0.17];
b2=[0.48];
eta=0.25;
figure(1);
clf;
clear nn_error;
x=-2:0.01:2;
gx=1+sin(x*pi/4);
h=plot(x,gx,'r--'); set(h,'LineWidth',[2]);
z=gx*0;
for i=1:length(x),
   n11=x(i)*W1(1,1)+b1(1,1);
   z11=1/(1+exp(0-n11));
   
   n12=x(i)*W1(2,1)+b1(2,1);
   z12=1/(1+exp(0-n12));
   
   z2=W2(1,1)*z11+W2(1,2)*z12+b2;
   z(i)=z2;
end
k = 0;
nn_error(k+1) = sum(abs(z-gx))/length(gx);
hold on;
h=plot(x,z); set(h,'LineWidth',[2]);
title([sprintf('Initial approximation (Average error %8.6f)',nn_error(1))]);
set(gcf,'Color',[1 1 1]);
pause;
while (sum(abs(z-gx)) > (0.0125*length(gx)))
   if k > 10000,
      break;
   end
   
   if k==0;
      x_i=1;
   else
      x_i=rand*4-2;
   end
   n11=x_i*W1(1,1)+b1(1,1);
   z11=1/(1+exp(0-n11));
   n12=x_i*W1(2,1)+b1(2,1);
   z12=1/(1+exp(0-n12));
   z2=W2(1,1)*z11+W2(1,2)*z12+b2;
   z_i=z2;
   t_i = 1+sin(x_i*pi/4);
   delta21 = (t_i-z_i);
   delta11 = (1-z11)*z11*W2(1,1)*delta21;
   delta12 = (1-z12)*z12*W2(1,2)*delta21;
   W2 = W2+eta*delta21*[z11 z12];
   b2 = b2+eta*delta21;
   W1(1,1)=W1(1,1)+eta*delta11*x_i;
   W1(2,1)=W1(2,1)+eta*delta12*x_i;
   b1 = b1 + eta*[delta11 delta12]';
   k=k+1;
   z=gx*0;
   for i=1:length(x),
      n11=x(i)*W1(1,1)+b1(1,1);
      z11=1/(1+exp(0-n11));
      
      n12=x(i)*W1(2,1)+b1(2,1);
      z12=1/(1+exp(0-n12));
      z2=W2(1,1)*z11+W2(1,2)*z12+b2;
      z(i)=z2;
   end
   nn_error(k+1) = sum(abs(z-gx))/length(gx);
   
   if mod((k-1), 100) == 0,
      hold off;
      h=plot(x,gx,'r--'); set(h,'LineWidth',[2]);
      
      hold on;
      h=plot(x,z); set(h,'LineWidth',[2]);
      
      title([sprintf('Average rror at %d iteration is %8.6f', k, nn_error(k+1))]);
      set(gcf,'Color',[1 1 1]);
      pause
   end
   %keyboard;  
end
z=gx*0;
for i=1:length(x),
   n11=x(i)*W1(1,1)+b1(1,1);
   z11=1/(1+exp(0-n11));
   
   n12=x(i)*W1(2,1)+b1(2,1);
   z12=1/(1+exp(0-n12));
   
   z2=W2(1,1)*z11+W2(1,2)*z12+b2;
   z(i)=z2;
end
%subplot(1,2,1);
figure(1);
hold off;
h=plot(x,gx,'r--'); set(h,'LineWidth',[2]);
hold on;
h=plot(x,z); set(h,'LineWidth',[2]);
title([sprintf('Final approximation (after %d iterations)',k)]);
set(gcf,'Color',[1 1 1]);
figure(2);
clf;
h=plot([1:(k+1)],nn_error); set(h,'LineWidth',[2]);
xlabel('Iterations');
ylabel('Average error');%
title('Average error');



