Upload 16 files

Data_Transformation/Run_Modularize_Data.m

@@ -0,0 +1,45 @@
+nmb_of_labels=1000;
+nmb_of_labs_per_module=25;
+nmb_of_modules=(nmb_of_labels/nmb_of_labs_per_module);
+relative_lab_seq=1:nmb_of_labs_per_module;
+nmb_of_subsets=2;
+
+patch=0;
+parfor module=1:nmb_of_modules
+    m_label_ids=[];
+    m_labels=[];
+    m_data=[];
+    m_label_table=[relative_lab_seq;relative_lab_seq+(module-1)*nmb_of_labs_per_module];
+    for imgnt1kdataset=1:10
+        % change the path to the folder containing the ImageNet-1k mat
+        % files
+        reportname1 = sprintf('/work/mathbiology/lheath2/data/imagenet1k/mat/train_data_batch_%d.mat', imgnt1kdataset);
+        temp_lpad=load(reportname1,'data','labels') %
+        data=temp_lpad.data;
+        labels=temp_lpad.labels;
+        pos_seq=1:length(labels);
+        for labs=relative_lab_seq
+            idx=(labels==(labs+(module-1)*nmb_of_labs_per_module));
+            aa=pos_seq(idx);
+            bb=[0*aa+imgnt1kdataset;aa;labels(idx)];
+            m_label_ids=[m_label_ids, bb];
+            m_labels=[m_labels,0*aa+labs];
+            m_data=[m_data;data(idx,:)];
+        end
+    end
+    nmb_dt=length(m_labels);
+    set_lng=fix(nmb_dt/nmb_of_subsets);
+    for subset=1:nmb_of_subsets
+        if subset<nmb_of_subsets
+            set=(1:set_lng)+(subset-1)*set_lng;
+        else
+            set=(1+(subset-1)*set_lng):nmb_dt;
+        end
+        data=m_data(set,:);
+        labels=m_labels(:,set);
+        label_ids=m_label_ids(:,set);
+        label_table=m_label_table;
+        out=fun_save_modularized_data(patch, module, subset,nmb_of_labs_per_module,data,labels,label_ids,label_table)
+    end
+    module
+end

Data_Transformation/Run_Transform_Data.m

@@ -0,0 +1,66 @@
+% clear all
+%
+nmb_of_modules=40;
+nmb_of_module_subsets=2;
+channels_names={'R','G','B','RGg1','RBg1','GBg1','RGg2','RBg2','GBg2','RB','RG','GB','eRGB','BW','X','Y','Z'};
+feature_RGB=[1 0 0
+    0      1      0
+    0      0      1
+    0.618  0.382  0
+    0.618  0      0.382
+    0      0.618  0.382
+    0.382  0.618  0
+    0.382  0      0.618
+    0      0.382  0.618
+    0.5    0.5    0
+    0.5    0      0.5
+    0      0.5    0.5
+    1/3    1/3    1/3    
+    0.299  0.587  0.114
+    0.4125 0.3576  0.1804
+    0.2126 0.7152  0.0722
+    0.0193 0.1192  0.9502]; 
+nmb_of_colors=length(channels_names);
+patch=0;
+nmb_of_labs_per_module=25;
+cross_entropy=1;
+
+param.image_size=[64,64];
+param.downsizing=2;
+param.x_trim=1;
+param.y_trim=1;
+param.compute_decimal_place=4;
+param.dwnsz_on=1;
+
+param.patch=patch;
+param.nmb_of_modules=nmb_of_modules;
+param.nmb_of_module_subsets=nmb_of_module_subsets;
+
+param.cross_entropy=cross_entropy;
+param.nmb_of_labs_per_module=nmb_of_labs_per_module;
+
+param.channels_names=channels_names;
+param.nmb_of_colors=nmb_of_colors;
+param.feature_RGB=feature_RGB;
+
+parfor module=1:nmb_of_modules
+    data_param=[];
+    for subset=1:nmb_of_module_subsets
+        reportname1 = sprintf('Transformed_IN1k_Data/Modularized_Data_for_SGD/modularized_data_patch_%d_module_%d_subset_%d_for_%d_labels_per_module.mat', ...
+            patch,module,subset,nmb_of_labs_per_module);
+        %     save(reportname1,'data','labels','label_ids','label_table','data_param');
+        data_load=load(reportname1);
+        data_0=data_load.data;
+        output=fun_transform_data_rgbfeatures(data_0,param);
+        data=output.transformed_image;
+        data_param.mnsv=output.mnsv;
+        data_param.maxsv=output.maxsv;
+        data_param.ipvsz=output.ipvsz;
+        labels=data_load.labels;
+        label_ids=data_load.label_ids;
+        label_table=data_load.label_table;
+        out=fun_save_transformed_data(patch, module, subset,nmb_of_labs_per_module,data,labels,label_ids,label_table,data_param);
+    end
+    module
+end
+%%

Data_Transformation/fun_save_modularized_data.m

@@ -0,0 +1,9 @@
+function out=fun_save_modularized_data(patch, module, subset,nmb_of_labs_per_module,data,labels,label_ids,label_table)
+%
+reportname1 = sprintf('Transformed_IN1k_Data/Modularized_Data_for_SGD/modularized_data_patch_%d_module_%d_subset_%d_for_%d_labels_per_module.mat', ...
+    patch,module,subset,nmb_of_labs_per_module);
+%     str=struct('data',data,'labels',labels,'label_ids',label_ids,'label_table',label_table,'data_param',data_param);
+%     save(reportname1,"-fromstruct",str);
+save(reportname1,'data','labels','label_ids','label_table');
+out=[];
+end

Data_Transformation/fun_save_transformed_data.m

@@ -0,0 +1,9 @@
+function out=fun_save_transformed_data(patch, module, subset,nmb_of_labs_per_module,data,labels,label_ids,label_table,data_param)
+%
+reportname1 = sprintf('Transformed_IN1k_Data/Transformed_Data_for_SGD/train_data_patch_%d_module_%d_subset_%d_for_%d_labels_per_module.mat', ...
+    patch,module,subset,nmb_of_labs_per_module);
+%     str=struct('data',data,'labels',labels,'label_ids',label_ids,'label_table',label_table,'data_param',data_param);
+%     save(reportname1,"-fromstruct",str);
+save(reportname1,'data','labels','label_ids','label_table','data_param');
+out=[];
+end

Data_Transformation/fun_transform_data_rgbfeatures.m

@@ -0,0 +1,93 @@
+function output=fun_transform_data_rgbfeatures(img_data,param)
+%
+%img_data=data_load.data;
+output.transformed_image=[];
+output.mnsv=[];
+output.maxsv=[];
+output.ipvsz=[];
+
+image_size=param.image_size;
+img_x_dim=image_size(1);
+img_y_dim=image_size(1);
+
+compute_decimal_place=param.compute_decimal_place;
+feature_RGB=param.feature_RGB;
+[nmb_of_colors,~]=size(feature_RGB);
+
+[data_size,~]=size(img_data);
+reshaped_images=zeros(data_size,img_x_dim,img_y_dim,3);
+for m=1:data_size
+    aa=double(img_data(m,:));
+    reshaped_images(m,:,:,:)=reshape(aa,img_x_dim,img_y_dim,3);
+end
+
+if param.dwnsz_on==1
+    x_trim=param.x_trim;
+    y_trim=param.y_trim;
+    downsizing=param.downsizing;
+    x_dwnsz_dim=fix(img_x_dim/downsizing);
+    y_dwnsz_dim=fix(img_y_dim/downsizing);
+
+    ipvsz=(x_dwnsz_dim-2*x_trim)*(y_dwnsz_dim-2*y_trim);
+    xsq=1:downsizing;
+    ysq=1:downsizing;
+    output.transformed_image=zeros(ipvsz,data_size,nmb_of_colors);
+    output.mnsv=zeros(data_size,nmb_of_colors);
+    output.maxsv=zeros(data_size,nmb_of_colors);
+    output.ipvsz=ipvsz;
+
+    temp_img=zeros(ipvsz,3);
+    for m=1:data_size
+        img=squeeze(reshaped_images(m,:,:,:));
+        for color=1:3
+            img1=squeeze(img(:,:,color));
+            aa=zeros(ipvsz,1);
+            for ii=(1+x_trim):(x_dwnsz_dim-x_trim)
+                for jj=(1+y_trim):(y_dwnsz_dim-y_trim)
+                    aa((jj-y_trim)+((ii-x_trim)-1)*(y_dwnsz_dim-2*y_trim),1)=mean(img1(xsq+2*(ii-1),ysq+2*(jj-1)),'all');
+                end
+            end
+            temp_img(:,color)=aa;
+        end
+        for color=1:nmb_of_colors
+            c1=feature_RGB(color,1);
+            c2=feature_RGB(color,2);
+            c3=feature_RGB(color,3);
+            aa=c1*temp_img(:,1)+c2*temp_img(:,2)+c3*temp_img(:,3);
+            mnsv=mean(aa);
+            aa=aa-mnsv;
+            maxsv=max(abs(aa));
+            output.transformed_image(:,m,color)=round(aa/maxsv*10^compute_decimal_place)*10^(-compute_decimal_place);
+            output.mnsv(m,color)=mnsv;
+            output.maxsv(m,color)=maxsv;
+        end
+    end
+else    
+    temp_img=zeros(img_x_dim*img_y_dim,3);
+    ipvsz=img_x_dim*img_y_dim;
+    output.ipvsz=ipvsz;
+    for m=1:data_size
+        img=squeeze(reshaped_images(m,:,:,:));
+        for color=1:3
+            aa=img(:,:,color);
+            aa=aa(:);
+            temp_img(:,color)=aa;
+        end
+        for color=1:nmb_of_colors
+            c1=feature_RGB(color,1);
+            c2=feature_RGB(color,2);
+            c3=feature_RGB(color,3);
+            aa=c1*temp_img(:,1)+c2*temp_img(:,2)+c3*temp_img(:,3);
+            mnsv=mean(aa);
+            aa=aa-mnsv;
+            maxsv=max(abs(aa));
+            output.transformed_image(:,m,color)=round(aa/maxsv*10^compute_decimal_place)*10^(-compute_decimal_place);
+            output.mnsv(m,color)=mnsv;
+            output.maxsv(m,color)=maxsv;
+        end             
+    end    
+end
+if data_size==1
+    output.transformed_image=squeeze(output.transformed_image);
+end
+end

Data_Transformation/read_me.m

@@ -0,0 +1,10 @@
+%
+% 1. Create two folders: 'Modularized_Data_for_SGD' and 
+%     'Transformed_Data_for_SGD' in the current foler. 
+%
+% 2. Run first 'Run_Modularize_Data.m' to organize the ImageNet-1k into 
+%     modules of labels. 
+%
+% 3. Run next 'Run_Transform_Data.m' to transform the modularized data set
+%     into the input data to the ANN models
+% 

Data_Transformation/run_script.m

@@ -0,0 +1,2 @@
+Run_Modularize_Data;
+Run_Transform_Data;

Run_Confusion_Matrix_Analysis.m

@@ -0,0 +1,83 @@
+% clear all
+
+proto_model={'T_h1_m1';'S_h1_m1';'T_h1_m2';'S_h1_m2';'T_h2_m1';'S_h2_m1'};
+nmb_of_proto_models=length(proto_model);
+featured_model={'model_1';'model_2';'model_3';'model_4';'model_5';'model_6'};
+[nmb_of_ft_models,~]=size(featured_model);
+
+nmb_of_lab=1000;
+nmb_of_batches=10;
+nmb_of_image_set=zeros(1,10);
+pr_set=zeros(1,10);
+model_accuracy_comparison=zeros(1,nmb_of_ft_models);
+pm.nmb_of_lab=nmb_of_lab;
+edges=1:1:(nmb_of_lab+1);
+% top_n_labels=10;
+confusion_matrixes=zeros(nmb_of_lab,nmb_of_lab,nmb_of_batches,nmb_of_ft_models,nmb_of_proto_models);
+model_c_matrixes=zeros(nmb_of_lab,nmb_of_lab,nmb_of_ft_models,nmb_of_proto_models);
+
+for ii=1:nmb_of_proto_models
+    md=char(proto_model(ii));
+%     confusion_matrixes=zeros(nmb_of_lab,nmb_of_lab,nmb_of_batches,nmb_of_ft_models);
+%     model_c_matrixes=zeros(nmb_of_lab,nmb_of_lab,nmb_of_ft_models);
+    top_100_pr_by_batch=zeros(nmb_of_batches,nmb_of_ft_models); % 100 postive rate
+    top_100_pr_by_model=zeros(1,nmb_of_ft_models); % 100 postive rate
+    batch_nmbs=zeros(1,nmb_of_batches);
+    for fm=1:nmb_of_ft_models
+        top_100_accnt=[];
+        c_matrix=zeros(nmb_of_lab,nmb_of_lab);
+        for imgnt1kdataset=1:nmb_of_batches
+            reportname1 = sprintf('Model_%s/Evaluation_Data/Model_Accuracy/training_data_batch_%d_feature_module_performance_%s_var.mat', md,imgnt1kdataset,md);
+            aa=sprintf('classification_data_%d',fm);
+            bb=load(reportname1,aa);
+            c_data=bb.(aa);
+            true_lab=c_data(:,1);
+            pred_lab=c_data(:,2);
+            nmb_of_data=length(true_lab);
+            c_mtx_output=fun_confusion_matrix(true_lab,pred_lab,pm);
+            bb=c_mtx_output.conf_matrix;
+            confusion_matrixes(:,:,imgnt1kdataset,fm,ii)=bb;
+            c_matrix=c_matrix+nmb_of_data*bb;
+            top_100_pr_by_batch(imgnt1kdataset,fm)=length(c_mtx_output.lab_100);
+            top_100_accnt=[top_100_accnt,c_mtx_output.lab_100];
+            batch_nmbs(imgnt1kdataset)=nmb_of_data;
+        end
+        model_c_matrixes(:,:,fm,ii)=c_matrix/sum(batch_nmbs);
+        histN = histcounts(top_100_accnt,edges);
+        idx=(histN==nmb_of_batches);
+        top_100_pr_by_model(fm)=sum(1*idx);
+    end
+%     mean(top_100_pr_by_batch,1) % average 100-rate by data batch
+%     top_100_pr_by_model
+    %%%%%%%
+    assignin('base',md, [mean(top_100_pr_by_batch,1);top_100_pr_by_model]')
+end
+%%
+table(featured_model,T_h1_m1,T_h1_m2,T_h2_m1)
+
+%% 
+% Find the labels that are perfectly classified, by model
+%
+ii=1;
+fm=6;
+lab=1:1000;
+A=model_c_matrixes(:,:,fm,ii);
+dd=diag(A,0);
+idx=(dd==100);
+lab_100=lab(idx)
+nmb_of_lab_100=sum(1*idx)
+
+%% 
+% Find the labels that are perfectly classified, by batch and model
+%
+ii=1;
+fm=6;
+imgnt1kdataset=1;
+lab=1:1000;
+A=confusion_matrixes(:,:,imgnt1kdataset,fm,ii);
+dd=diag(A,0);
+idx=(dd==100);
+lab_100=lab(idx)
+nmb_of_lab_100=sum(1*idx)
+
+

Run_Model_Analysis.m

@@ -0,0 +1,93 @@
+% clear all
+
+proto_model={'T_h1_m1';'S_h1_m1';'T_h1_m2';'S_h1_m2';'T_h2_m1';'S_h2_m1'};
+nmb_of_proto_models=length(proto_model);
+featured_model={'model_1';'model_2';'model_3';'model_4';'model_5';'model_6'};
+[nmb_of_ft_models,~]=size(featured_model);
+for ii=1:nmb_of_proto_models
+    md=char(proto_model(ii));
+    %%%%%%%
+    nmb_of_image_set=zeros(1,10);
+    pr_set=zeros(1,10);
+    top_1_set=zeros(1,10);
+    model_accuracy_comparison=zeros(2,nmb_of_ft_models);
+    for fm=1:nmb_of_ft_models
+        for imgnt1kdataset=1:10
+            reportname1 = sprintf('Model_%s/Evaluation_Data/Model_Accuracy/training_data_batch_%d_feature_module_performance_%s_var.mat',...
+                md,imgnt1kdataset, md);
+            aa=sprintf('classification_data_%d',fm);
+            bb=load(reportname1,aa);
+            c_data=bb.(aa);
+            true_lab=c_data(:,1);
+            pred_lab=c_data(:,2);
+            likelyhood=c_data(:,3);
+            top_1_majority=c_data(1,3);
+            nmb_of_images=length(true_lab);
+            nmb_of_image_set(imgnt1kdataset)=nmb_of_images;
+            idx=(abs(true_lab-pred_lab)==0);
+            aa=sum(1*idx);
+            pr=aa/nmb_of_images*100;
+            pr_set(imgnt1kdataset)=pr;
+            %%%%%%%% Top-1 rate %%%%%%%%%%%
+            bb=likelyhood(idx);
+            cc=length(bb);
+            idx1=(bb==top_1_majority);
+            aa=sum(idx1*1);
+            top_1=aa/cc*100;
+            top_1_set(imgnt1kdataset)=top_1;
+        end
+        model_accuracy_comparison(1,fm)=nmb_of_image_set*pr_set'/sum(nmb_of_image_set);
+        model_accuracy_comparison(2,fm)=nmb_of_image_set*top_1_set'/sum(nmb_of_image_set);
+%         assignin('base',featured_model(fm), model_accuracy_comparison')
+    end
+assignin('base',md, model_accuracy_comparison')
+end
+
+%%
+tb1=table(featured_model,T_h1_m1,S_h1_m1,T_h1_m2,S_h1_m2,T_h2_m1,S_h2_m1)
+
+%%
+table(featured_model,T_h1_m1,T_h1_m2,T_h2_m1)
+
+table(featured_model,S_h1_m1,S_h1_m2,S_h2_m1)
+
+%%
+for fm=1:nmb_of_ft_models
+    %%%%%%%
+    nmb_of_image_set=zeros(1,10);
+    pr_set=zeros(1,10);
+    top_1_set=zeros(1,10);
+    model_accuracy_comparison_2=zeros(2,nmb_of_ft_models);
+    for ii=1:nmb_of_proto_models
+        md=char(proto_model(ii));
+        for imgnt1kdataset=1:10
+            reportname1 = sprintf('Model_%s/Evaluation_Data/Model_Accuracy/training_data_batch_%d_feature_module_performance_%s_var.mat',...
+                md,imgnt1kdataset, md);
+            aa=sprintf('classification_data_%d',fm);
+            bb=load(reportname1,aa);
+            c_data=bb.(aa);
+            true_lab=c_data(:,1);
+            pred_lab=c_data(:,2);
+            likelyhood=c_data(:,3);
+            top_1_majority=c_data(1,3);
+            nmb_of_images=length(true_lab);
+            nmb_of_image_set(imgnt1kdataset)=nmb_of_images;
+            idx=(abs(true_lab-pred_lab)==0);
+            aa=sum(1*idx);
+            pr=aa/nmb_of_images*100;
+            pr_set(imgnt1kdataset)=pr;
+            %%%%%%%% Top-1 rate %%%%%%%%%%%
+            bb=likelyhood(idx);
+            cc=length(bb);
+            idx1=(bb==top_1_majority);
+            aa=sum(idx1*1);
+            top_1=aa/cc*100;
+            top_1_set(imgnt1kdataset)=top_1;
+        end
+        model_accuracy_comparison_2(1,ii)=nmb_of_image_set*pr_set'/sum(nmb_of_image_set);
+        model_accuracy_comparison_2(2,ii)=nmb_of_image_set*top_1_set'/sum(nmb_of_image_set);
+    end
+    assignin('base',char(featured_model(fm)), round(model_accuracy_comparison_2,3)')
+end
+
+tb2=table(proto_model,model_1,model_2,model_3,model_4,model_5,model_6)

Run_Spoke_Plot.m

@@ -0,0 +1,40 @@
+% clear all
+
+channels_names={'R','G','B','RGg1','RBg1','GBg1','RGg2','RBg2','GBg2','RB','RG','GB','eRGB','BW','X','Y','Z'};
+patch=0;
+module=9;%4;%38;
+subset=2;
+color=4;
+nmb_of_labs_per_module=25;
+nmb_of_labels=nmb_of_labs_per_module;
+maxlngth=3000;
+
+reportname1 = sprintf('Data_Transformation/Transformed_IN1k_Data/Transformed_Data_for_SGD/train_data_patch_%d_module_%d_subset_%d_for_%d_labels_per_module.mat', ...
+    patch,module,subset,nmb_of_labs_per_module);
+data_load=load(reportname1);
+vect_image=data_load.data(:,1:maxlngth,color);
+true_label=data_load.labels(1:maxlngth);
+
+model='T_h2_m1';
+reportname1 = sprintf(['Model_%s/Model_Parameter/Trained_Parameter_patch_%d_module_%d_subset_%d_ch_%s.mat'],...
+    model,patch, module, subset, char(channels_names(color)));
+%      str=struct('W', W, 'b', b);
+temp_load=load(reportname1);
+Wt=temp_load.W;
+bt=temp_load.b;
+reportname1 = sprintf('Model_%s/Training_Evaluation/%s_1_performance.mat',model,model);
+load(reportname1)
+tpr=pstvrt_model(color,module,subset);
+
+model='S_h2_m1';
+reportname1 = sprintf('Model_%s/Model_Parameter/Trained_Parameter_patch_%d_module_%d_subset_%d_ch_%s.mat',...
+    model, patch, module, subset, char(channels_names(color)));
+%      str=struct('W', W, 'b', b);
+temp_load=load(reportname1);
+Ws=temp_load.W;
+bs=temp_load.b;
+reportname1 = sprintf('Model_%s/Training_Evaluation/%s_1_performance.mat',model,model);
+load(reportname1)
+spr=pstvrt_model(color,module,subset);
+
+fun_spoke_plot(vect_image,true_label,Ws,bs,spr,Wt,bt,tpr,nmb_of_labels)

Run_Training_Analysis.m

@@ -0,0 +1,35 @@
+% clear all
+
+model={'T_h1_m1';'S_h1_m1';'T_h1_m2';'S_h1_m2';'T_h2_m1';'S_h2_m1';};
+nmb_of_models=length(model);
+rtmax=[];
+rtmin=[];
+rtmed=[];
+rtmean=[];
+nmb_of_100rt_module=[];
+nmb_of_100rt=[];
+stat_100rt=[];
+for ii=1:nmb_of_models
+    md=char(model(ii));
+    reportname1 = sprintf('Model_%s/Training_Evaluation/%s_1_performance.mat',md,md);
+    load(reportname1)
+
+    rtmax=[rtmax;max(pstvrt_model,[],'all')];
+    rtmin=[rtmin;min(pstvrt_model,[],'all')];
+    rtmed=[rtmed;median(pstvrt_model(:),'all')];
+    rtmean=[rtmean;mean(pstvrt_model(:),'all')];
+
+    aa=squeeze(pstvrt_model(2,:,1));
+    bb=squeeze(pstvrt_model(2,:,2));
+    cc=[aa,bb];
+    idx=(cc==100);
+    nmb_of_100rt_module=[nmb_of_100rt_module;sum(1*idx)];
+    %     rt100=(sum(idx*1)-1)/length(cc);
+    dd=[squeeze(pstvrt_model(:,:,1));squeeze(pstvrt_model(:,:,2))];
+    idx2=(dd==100);
+    nmb_of_100rt=[nmb_of_100rt;sum(1*idx)];
+    stat_100rt=[stat_100rt;[sum(1*idx2,'all'),sum(1*idx)]];
+end
+%%
+
+perfmn=table(model,rtmin,rtmean,rtmed,rtmax,stat_100rt)

fun_activation.m

@@ -0,0 +1,21 @@
+function [out1,out2]=fun_activation(x)
+%
+%
+%
+[sz1,sz2]=size(x);
+A=zeros(sz1,sz2);
+B=zeros(sz1,sz1,sz2);
+for zz=1:sz2
+    [A(:,zz),B(:,:,zz)]=ReLU(x(:,zz));
+end
+
+out1=A;
+out2=B;
+%%%%%%%%
+    function [y,dy]=ReLU(s)
+        aa=(s>0).*1;
+        y=aa.*s;
+        dy=diag(aa);
+    end
+%%%%%%%%
+end

fun_confusion_matrix.m

@@ -0,0 +1,30 @@
+function output=fun_confusion_matrix(true_lab,pred_lab,pm)
+%
+nmb_of_images=length(true_lab);
+nmb_of_lab=pm.nmb_of_lab;
+% top_n=pm.top_n;
+edges=1:1:(nmb_of_lab+1);
+c_matrix=zeros(nmb_of_lab,nmb_of_lab);
+seq_lab=1:nmb_of_lab;
+% top_n_seq=1:top_n;
+p_seq=(1:nmb_of_images)';
+for lb=1:nmb_of_lab
+    idx1=(true_lab==lb);
+    aa=p_seq(idx1);
+    nmb=length(aa);
+    aa=pred_lab(aa);
+    [N,~]=histcounts(aa,edges);
+    rt=N/nmb*100;
+    c_matrix(lb,:)=rt;
+end
+aa=diag(c_matrix);
+% [~,cc]=sort(aa);
+idx100=(aa==100);
+output.lab_100=seq_lab(idx100);
+% idx=flip(cc);
+% c_matrix=c_matrix(idx, idx);
+output.conf_matrix=c_matrix;
+% top_n_lb=idx(top_n_seq);
+% top_n_rt=diag(c_matrix(top_n_seq,top_n_seq));
+% output.top_n_inf=[top_n_rt,top_n_lb];
+end

fun_predicted_vector_2_label.m

@@ -0,0 +1,36 @@
+function out=fun_predicted_vector_2_label(vector,nmb_of_labels)
+%
+% predicted vector of length 25 to digit label 1, ..., 25
+% vector=predicted_vector;
+[~,dtsz]=size(vector);
+digit=zeros(1,dtsz);
+distance=zeros(1,dtsz);
+nn=nmb_of_labels;
+v=vector(1:nn,:);
+I=eye(nn);
+vv=zeros(nn,1);
+for j=1:dtsz
+    for i=1:nn
+        [cc,~]=loss_function(v(:,j),I(:,i));
+        vv(i)=cc;
+    end
+    [aa,idx]=min(vv);
+    digit(j)=idx;
+    distance(j)=aa;
+end
+out.label=digit;
+out.distance=distance;
+%%%%%%%%%%
+    function [loss,dloss]=loss_function(predicted_x,true_y)
+        % cross_entropy loss
+        %
+        eps=1e-8;
+        [n, N]=size(true_y);
+        loss=sum(sum((true_y+eps).*log((true_y+eps)./(predicted_x+eps))))/N;
+        %         ind_loss=sum((true_y+eps).*log((true_y+eps)./(predicted_x+eps)),1);
+        bb=-(true_y(1:end-1,:)+eps)./(predicted_x(1:end-1,:)+eps);
+        w=bb+ones(n-1,1)*(true_y(end,:)+eps)./(predicted_x(end,:)+eps);
+        dloss=w'/N;
+    end
+%%%%%%%%%%
+end

fun_softmax.m

@@ -0,0 +1,39 @@
+function [out1,out2]=fun_softmax(S)
+%
+% Column-wise normaliztion
+%
+[sz1,sz2]=size(S);
+A=zeros(sz1,sz2);
+B=zeros(sz1-1,sz1,sz2);
+for zz=1:sz2
+    [aaa,bbb]=softmax(S(:,zz));
+    A(:,zz)=aaa;
+    B(:,:,zz)=bbb;
+end
+out1=A;
+out2=B;
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    function [y,dy]=softmax(S)
+        %         bb=1e+2;
+        bb=1;
+        S=S/bb;
+        aa=exp(S);
+        sm=sum(exp(S))+1e-16;
+        y=aa./sm;
+
+        [n,~]=size(S);
+        m=n-1;
+        dy=zeros(m,n);
+        for i=1:m
+            for j=1:n
+                if j~=i
+                    dy(i,j)=-exp(S(i)).*exp(S(j));
+                else
+                    dy(i,j)=exp(S(i)).*sm-exp(S(i)).*exp(S(j));
+                end
+            end
+        end
+        dy=dy/sm^2/bb;
+    end
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+end

fun_spoke_plot.m

@@ -0,0 +1,160 @@
+function []=fun_spoke_plot(vect_image,true_label,Ws,bs,spr,Wt,bt,tpr,nmb_of_labels)
+%
+%
+lb=true_label;
+nml=nmb_of_labels;
+nn=3;
+clr=colormap(cool(nn*(nml+1)));
+lbnm={'a','b','c','d','e','f','g','h','i','j','k','l','m','n',...
+    'o','p','q','r','s','t','u','v','w','x','y','z'};
+gy=.33;
+grdcl='w';
+grlwt=2;
+msz=4;
+redge=1.1;
+
+figure(1)
+hold off
+
+subplot(1,2,1)
+prediction=fun_prediction(vect_image,Ws,bs);
+predicted=prediction.A_end;
+[~,nmd]=size(predicted);
+offst=.07;
+%%%%%%%%%%%%%%%
+inc=2*pi/nml;
+tht=(0:(nml-1))*inc;
+A=zeros(2,nmb_of_labels);
+for ii=1:nml %tht
+    A(:,ii)=[tht(ii);1];
+end
+radii=1./abs(prediction.predicted.distance+offst);
+%%%%%%%%%%%%%%%
+pi_pt=zeros(2,nmd);
+for mm=1:nmd
+    pi_pt(:,mm)=A*predicted(:,mm);
+end
+
+aa=pi_pt(1,:);
+theta_all=aa(:);
+% aa=pi_pt(2,:);
+% rho_all=aa(:);
+%%%%%%%%%%%%%%%
+% polarplot(twopi,unitclc,'linewidth',1,'Color','w');
+% hold on
+for kk=1:nml
+    idx=(lb==kk);
+    theta=theta_all(idx);
+    rho=radii(idx);
+    % rho=rho_all(idx);
+    p=polarplot(theta,rho/max(rho));
+    hold on
+
+    p.Marker = 'square';
+    p.MarkerSize = msz;
+    p.LineStyle = "none";
+    p.Color = clr(nn*kk,:);
+    p.MarkerFaceColor = clr(nn*kk,:);
+end
+ax = gca;
+ax.RTickLabel = {};
+ax.ThetaTick = rad2deg(tht);
+ax.ThetaTickLabel = lbnm;
+axis([-inf, inf, 0,redge])
+set(ax,'Color',[gy gy gy])
+set(ax,'GridColor',grdcl,'LineWidth',grlwt)
+
+% title("SGD - trained Model (" + SGDpr*epoch_size_SGD/epoch_sz_GDT + "%)",'fontsize',14)
+title("SGD - trained Model (" + round(spr,2) + "%)",'fontsize',14)
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+subplot(1,2,2)
+prediction=fun_prediction(vect_image,Wt,bt);
+predicted=prediction.A_end;
+[~,nmd]=size(predicted);
+% offst=.05;
+%%%%%%%%%%%%%%%
+inc=2*pi/nml;
+tht=(0:(nml-1))*inc;
+A=zeros(2,nmb_of_labels);
+for ii=1:nml %tht
+    A(:,ii)=[tht(ii);1];
+end
+radii=1./abs(prediction.predicted.distance+offst);
+%%%%%%%%%%%%%%%
+pi_pt=zeros(2,nmd);
+for mm=1:nmd
+    pi_pt(:,mm)=A*predicted(:,mm);
+end
+
+aa=pi_pt(1,:);
+theta_all=aa(:);
+% aa=pi_pt(2,:);
+% rho_all=aa(:);
+%%%%%%%%%%%%%%%
+% polarplot(twopi,unitclc,'linewidth',1,'Color','w');
+% hold on
+for kk=1:nml
+    idx=(lb==kk);
+    theta=theta_all(idx);
+    rho=radii(idx);
+    % rho=rho_all(idx);
+    p=polarplot(theta,rho/max(rho));
+    hold on
+
+    p.Marker = 'square';
+    p.MarkerSize = msz;
+    p.LineStyle = "none";
+    p.Color = clr(nn*kk,:);
+    p.MarkerFaceColor = clr(nn*kk,:);
+end
+ax = gca;
+ax.RTickLabel = {};
+ax.ThetaTick = rad2deg(tht);
+ax.ThetaTickLabel = lbnm;
+axis([-inf, inf, 0,redge])
+set(ax,'Color',[gy gy gy])
+set(ax,'GridColor',grdcl,'LineWidth',grlwt)
+
+% title('Fully Trained Model (100%)','fontsize',14)
+title("GDT - trained Model (" + tpr + "%)",'fontsize',14)
+
+set(gcf,'Position',[10 80 900 440])
+% set(gcf, 'MenuBar', 'None')
+% sgtitle('Spoke-Plot for Training', 'fontsize', 16)
+sgtitle('Confusion Wheel for Training', 'fontsize', 16)
+
+    function out=fun_prediction(vect_image,W, b)
+        %
+        %
+        nmb_of_hidden_layers=length(fieldnames(W))-1;
+        W1=W.LayerName1;
+        W2=W.LayerName2;
+        b1=b.LayerName1;
+        b2=b.LayerName2;
+
+        a_0=vect_image;
+        %             true_label=data_load.labels;
+        %             dtsz=length(true_label);
+        nmb_labels=length(b2);
+        z1=W1*a_0+b1;
+        [a1,~]=fun_activation(z1);
+        z2=W2*a1+b2;
+
+        if nmb_of_hidden_layers==1
+            [a2,~]=fun_softmax(z2);
+            predicted_vector=a2;
+        else
+            W3=W.LayerName3;
+            b3=b.LayerName3;
+            nmb_labels=length(b3);
+            [a2,~]=fun_activation(z2);
+
+            z3=W3*a2+b3;
+            [a3,~]=fun_softmax(z3);
+            predicted_vector=a3;
+        end
+        out.A_end=predicted_vector;
+        out.predicted=fun_predicted_vector_2_label(predicted_vector,nmb_labels);
+    end
+end