ANN로 headPoseEstimate하기

1. Dataset

 (위와 같은 50*50 사진이 training에는 11275개, test에는 5000개가 있다.)

QMULPoseHeads.z01

QMULPoseHeads.z02

QMULPoseHeads.zip

2. Matlab code

 1. 각 5개 카테고리(front, back, left, right, non-face)의 data를 읽는다. 이 data는 50*50=2500의 rgb값이다. 

 2. rgb를 grayscale로 변환한 후에 ANN train할 수 있는 data로 구조를 맞춘다. 

 3. 2500 dimension은 너무 크기 때문에 PCA를 통해서 100 dimension으로 낮춘다. 

 4. 이렇게 구해진 data set과 label로 ANN을 train한다. 

 5. test data 역시 읽어 온 후에 PCA Matrix W를 곱해서 100 dimension으로 낮추고, ANN simulation한다. 

 6. 필요한 workspace의 data를 저장한다. 

%% Clear all

clc;

clear all;

close all;

warning off;

%% First, get training data from the folder

fprintf('1. Get training data from the folder\r\n')

%dirRootPath = 'C:/Users/CPSLab/Dropbox/연구/CPSLab/QMULPoseHeads/QMULPoseHeads';  % PC in CPS Lab 

dirRootPath = 'C:/Users/Human/Dropbox/연구/CPSLab/QMULPoseHeads/QMULPoseHeads'; % ma XNote Laptop

% For initialization

trainFeatureCount   = 50*50;            % the size of the image is 50 by 50                 

trainDataCount      = 1500;             % total: 2255 - but it will produce out-of-memory error.. I need 64bit MATLAB :(

trainTargetCount    = 5;                % there are total five categories; front, back, left, right face and non-face

% make space for training data (the size is defined above)

trainData           = zeros(trainFeatureCount, trainDataCount*trainTargetCount);

trainTargets        = zeros(trainTargetCount, trainDataCount*trainTargetCount);

% Set Training data

iprev = 0;

for i = 1:5

    % for every five categories

    % Initialize category 

    if iprev ~= i

        iprev = i;

        if i==1

            fprintf('\r\nfront face \t > ');

            dirPath     = sprintf('%s/train/Data_f', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

            target = [1 0 0 0 0]';

        elseif i==2

            fprintf('\r\nback face \t > ');

            dirPath     = sprintf('%s/train/Data_b', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

            target = [0 1 0 0 0]';

        elseif i==3

            fprintf('\r\nleft face \t > ');

            dirPath     = sprintf('%s/train/Data_l', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

            target = [0 0 1 0 0]';

        elseif i==4

            fprintf('\r\nright face \t > ');

            dirPath     = sprintf('%s/train/Data_r', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

            target = [0 0 0 1 0]';

        elseif i==5

            fprintf('\r\nnon face \t > ');

            dirPath     = sprintf('%s/train/Data_bg', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

            target = [0 0 0 0 1]';

        end

    end

    

    k = 1;

    for j = 1 : trainDataCount        

        % for every images(five categories) in the category

        image = imread( sprintf('%s/%s', dirPath, fileNames{j} ) );

        Igray = rgb2gray(image);        

        Igray = reshape( Igray, 50*50, 1);

        Igray = Igray - min(Igray);   %%%%%%%%%%    

        trainData( : , (i-1)*trainDataCount+j )    = Igray;

        trainTargets( : , (i-1)*trainDataCount+j ) = target;

            

        % make 10 dots total. 

        if rem(j, int16(trainDataCount/10) ) == 0

            fprintf('%d ', k);  

            figure(1);

            subaxis(10, 5, (i-1)*10 + k, 'Spacing', 0.03, 'Padding', 0, 'Margin', 0);

            %subplot(10, 5, (i-1)*10 + k);

            imshow( reshape(Igray, 50, 50));

            

            if i==1

                imgTitle = sprintf('front train %d', k);

            elseif i== 2

                imgTitle = sprintf('back train %d', k);

            elseif i== 3

                imgTitle = sprintf('left train %d', k);

            elseif i== 4

                imgTitle = sprintf('right train %d', k);

            elseif i== 5

                imgTitle = sprintf('non train %d', k);

            end

            title(imgTitle);

            k = k + 1;

        end

    end

end

fprintf('\r\n');

fprintf('starting PCA...');

W = princomp(trainData');

fprintf('...Done \r\n');

W = W';

W = W(1 : 100, :);         % until n-th principle component. 

trainData = W*trainData;

%% Second, Neural Network train

hiddenLayerCount = 50;

outputLayerCount = size(trainTargets, 1);

net = newff( minmax(trainData), [hiddenLayerCount outputLayerCount], ...

    {'tansig' 'tansig'}, 'traingdx' );

%net.LW{2, 1}    = 0.1*net.LW{2, 1};

%net.b{2,1}      = 0.1*net.b{2,1};

net.performFcn  = 'mse';

net.trainParam.goal = 0.001;

net.trainParam.show = 20;

net.trainParam.epochs = 20000;

net.trainParam.mc = 0.95;

% Actual, training (takes a lot of time)

fprintf('2. Train Neural Network\r\n')

[net, tr] = train(net, trainData, trainTargets);

%% Test trained Neural net

fprintf('3. Get test data\r\n')

testFeatureCount   = 50*50;

testDataCount      = 1000; %1000;

testTargetCount    = 5;

testData           = zeros(testFeatureCount, testDataCount*testTargetCount);

iprev        = 0;

for i = 1:5

    % for every five categories

    % Initialize category 

    if iprev ~= i

        iprev = i;

        if i==1

            fprintf('\r\nfront face \t > ');

            dirPath     = sprintf('%s/test/Test_f', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

        elseif i==2

            fprintf('\r\nback face \t > ');

            dirPath     = sprintf('%s/test/Test_b', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

        elseif i==3

            fprintf('\r\nleft face \t > ');

            dirPath     = sprintf('%s/test/Test_l', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

        elseif i==4

            fprintf('\r\nright face \t > ');

            dirPath     = sprintf('%s/test/Test_r', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

        elseif i==5

            fprintf('\r\nnon face \t > ');

            dirPath     = sprintf('%s/test/Test_bg', dirRootPath);

            fileList    = dir( sprintf('%s/*.jpg',dirPath) );

            fileNames   = {fileList.name};

        end

    end

    k = 1;

    for j = 1 : testDataCount

        

        % for every images(five categories) in the category

        image = imread( sprintf('%s/%s', dirPath, fileNames{j} ) );

        Igray = rgb2gray(image);  

        %imshow(Igray);

        Igray = reshape( Igray, testFeatureCount, 1);

        Igray = Igray - min(Igray);   %%%%%%%%%%           

        testData( : , (i-1)*testDataCount+j )    = Igray;

        

        % 10 counts 

        if rem(j, int16(testDataCount/10) ) == 0

            fprintf('%d ', k);   

            figure(2);

            %subplot(10, 5, (i-1)*10 + k);

            subaxis(10, 5, (i-1)*10 + k, 'Spacing', 0.03, 'Padding', 0, 'Margin', 0);

            imshow( reshape(Igray, 50, 50));

            if i==1

                imgTitle = sprintf('front test %d', k);

            elseif i== 2

                imgTitle = sprintf('back test %d', k);

            elseif i== 3

                imgTitle = sprintf('left test %d', k);

            elseif i== 4

                imgTitle = sprintf('right test %d', k);

            elseif i== 5

                imgTitle = sprintf('non test %d', k);

            end

            title(imgTitle);

              k = k + 1;

        end

    end

end

fprintf('\r\n');

%% test ANN trained above. 

[a, b] = max(sim(net, W*testData));

% get overall results

fRate = histc( b(1:testDataCount), 1)/testDataCount;

bRate = histc( b(testDataCount+1:testDataCount*2), 2)/testDataCount;

lRate = histc( b(testDataCount*2+1:testDataCount*3), 3)/testDataCount;

rRate = histc( b(testDataCount*3+1:testDataCount*4), 4)/testDataCount;

bgRate = histc( b(testDataCount*4+1:testDataCount*5), 5)/testDataCount;

% print overall results

fprintf('front detection rate: %.1f , certainty: %.1f   \r\n', fRate*100, mean(a(1:testDataCount))*100 );

fprintf('back detection rate: %.1f , certainty: %.1f   \r\n', bRate*100, mean(a(testDataCount+1:testDataCount*2)) *100);

fprintf('left detection rate: %.1f , certainty: %.1f    \r\n', lRate*100, mean(a(testDataCount*2+1:testDataCount*3))*100 );

fprintf('right detection rate: %.1f , certainty: %.1f    \r\n', rRate*100, mean(a(testDataCount*3+1:testDataCount*4))*100 );

fprintf('non-face detection rate: %.1f , certainty: %.1f    \r\n', bgRate*100, mean(a(testDataCount*4+1:testDataCount*5))*100 );

%% Save workspace

save('headPoseEstimateUsingANN.mat', 'net', 'W', 'trainData', 'trainTargets', 'testData');

3. Simulation result

 

 이 결과는 논문에서 나온 평균 80%와 비슷한 수치이다. 즉 ANN을 좀 더 tune하고, feature extraction을 제대로 할 경우 더 좋은 결과를 기대할 수도 있을 것이다. 

OrozcoGongXiang_BMVC09.pdf

 

4. Zip archive

Zip archive contains 

 1. Matlab source

 2. Workspace (net, W(PCA Matrix), trainData, trainTargets, testData)

headPoseEstimationUsingANN(pw_ adsl_2).z01

headPoseEstimationUsingANN(pw_ adsl_2).zip