Object detection/ segmentation OpenCV C++

!!! Please tell me what information you need !!! I also included the header files now.

I have to submit this project in a few days and need help desperately. I have to write a program which can detect the different nuts in the picture and classify them.

First I did a background segmentation, but I don't know which values to use, so that it functions for all pictures. I used floodfill and threshold for binarization and got rid of the smaller pixels. (The code is down below)

Now what needs to be done is:

1.) Choose characteristics to describe the segments in the picture. Possible characteristics are e.g.:

- min./ max. expansion in x- or y-direction of a segment

- area of a segment

- center point of a segment

- contour/ outline of a segment

- average min./max. colour of a segment

For the practical implementation of this step, the Class segment can be extended by additional class attributes for the respective characteristics.

2.) Teaching the classifier by the user

Next, the classifier is taught. This means that the user selects some examples for the various object types on the screen and thus defines the so-called 'training set', which in the following is used to classify the remaining segments (test set). The 'segPickHandler' is already prepared for this in the code. The function provides the basic structure for the user selection of the segments by mouse click and must be supplemented accordingly. To assign a segment to an object type, the attribute 'group' of the class 'Segment' should be used. After learning the classifier, the actual Classification is performed.

3.) Classification of segmented objects

After the training set has been filled with training objects, the objects that have not been marked should be assigned to one of the available object classes. The characteristics should be chosen so that a clear distinction of the different classes is possible. The class 'Classifier' provides the framework for implementing a Classifier. From this base-class further classes are to be derived, which implement concrete classifiers.

The following classifiers should be implemented:

- nearest-neighbour classifier

- k-nearest-neighbour classifier

- linear classifier

So basically the programm should be able to distinguish between peanuts, cashew nut and raisins by mouse click and then depending on what I want fill it in with e.g. red colour.

Here the pictures which should be used:

Classifier.h

#pragma once

#include 

#include 

#include "Segment.h"

class Classifier

{

protected:

 std::vector trainingSet_;

 std::vector testSet_;

public:

 Classifier(void);

 virtual ~Classifier(void);

 virtual void addToTrainingSet( Segment* seg );

 virtual void addToTestSet( Segment* seg );

 virtual void clearTrainingSet();

 virtual void clearTestSet();

 virtual std::vector& getTrainingSet();

 virtual std::vector& getTestSet();

 virtual bool classify() = 0;

};

Segment.h

#pragma once

#include 

class Segment

{

public:

 enum Group

 {

 A = 1,

 B = 2,

 C = 3,

 D = 4,

 E = 5

 };

 struct PixelCoordinate

 {

 PixelCoordinate() {};

 PixelCoordinate( int x_, int y_ ) { x = x_; y = y_; };

 int x;

 int y;

 };

 Segment(void);

 virtual ~Segment(void);

 int id_;

 bool valid_;

 bool assigned_;

 int group_;

 int numPixels_;

 std::vector pixels_;

 /// Features ///

 /// Methods to calculate the features (optional). ///

 // ...

};

Classifier_kNN.h

#pragma once

#include "Classifier.h"

class Classifier_kNN : public Classifier

{

public:

 Classifier_kNN(void);

 virtual ~Classifier_kNN(void);

 virtual bool classify();

};

Segment.cpp

#include "Segment.h"

Segment::Segment(void)

{

 id_ = -1;

 valid_ = false;

 assigned_ = false;

 group_ = -1;

 numPixels_ = 0;

}

Segment::~Segment(void)

{

 pixels_.clear();

}

Classifier.cpp

#include "Classifier.h"

Classifier::Classifier(void)

{

}

Classifier::~Classifier(void)

{

}

void Classifier::addToTrainingSet( Segment* seg )

{

 trainingSet_.push_back( seg );

}

void Classifier::addToTestSet( Segment* seg )

{

 testSet_.push_back( seg );

}

void Classifier::clearTrainingSet()

{

 trainingSet_.clear();

}

void Classifier::clearTestSet()

{

 testSet_.clear();

}

std::vector& Classifier::getTrainingSet()

{

 return trainingSet_;

}

std::vector& Classifier::getTestSet()

{

 return testSet_;

}

Classifier_kNN.cpp

#include "Classifier_kNN.h"

Classifier_kNN::Classifier_kNN(void)

{

}

Classifier_kNN::~Classifier_kNN(void)

{

}

bool Classifier_kNN::classify()

{

 std::cerr  
 std::cerr  
 
 
 if ( (trainingSet_.size() == 0) || (testSet_.size() == 0) ) { std::cerr  
 
 
 
 
 /// Do classification. ///
 
 // ...
 
 
 
 
 
 return true;
 
}
 
main.cpp
 
#include 
 
#include 
 
#include 
 
#include 
 
#include 
 
 
 
#include 
 
#include 
 
 
 
#include "Segment.h"
 
#include "Classifier_kNN.h"
 
 
 
using namespace cv;
 
using namespace std;
 
 
 
 
 
/*** STRUCT DEFINES ***/
 
 
 
struct RgbPixel
 
{
 
 RgbPixel() {};
 
 RgbPixel(int r_, int g_, int b_) { r=r_; g=g_; b=b_; };
 
 
 
 int r;
 
 int g;
 
 int b;
 
};
 
 
 
/*** HELPER FUNCTIONS ***/
 
 
 
bool isEqual(RgbPixel px1, RgbPixel px2)
 
{
 
 return ( (px1.r == px2.r) && (px1.g == px2.g) && (px1.b == px2.b) );
 
}
 
 
 
RgbPixel getRgbPixel(int x /*column*/, int y /*row*/, cv::Mat* img)
 
{
 
 Vec3b rgb = img->at(y, x);
 
 RgbPixel px;
 
 px.r = (int)rgb[2];
 
 px.g = (int)rgb[1];
 
 px.b = (int)rgb[0];
 
 return px;
 
}
 
 
 
void setRgbPixel(int x /*column*/, int y /*row*/, RgbPixel pxRgb, cv::Mat* img)
 
{
 
 img->at(y, x)[2] = (uchar) pxRgb.r;
 
 img->at(y, x)[1] = (uchar)pxRgb.g;
 
 img->at(y, x)[0] = (uchar)pxRgb.b;
 
 /*CvScalar s;
 
 s.val[2] = (double)pxRgb.r;
 
 s.val[1] = (double)pxRgb.g;
 
 s.val[0] = (double)pxRgb.b;
 
 cvSet2D(img,y,x,s); */
 
}
 
 
 
int convertRgbToGray(RgbPixel px)
 
{
 
 double fGray = 0.3*(double)px.r + 0.59*(double)px.g + 0.11*(double)px.b;
 
 int iGray = (int)fGray;
 
 return iGray;
 
}
 
 
 
/*** INPUT HANDLER ***/
 
 
 
struct ImgSegSet
 
{
 
 Mat* img;
 
 vector* segVec;
 
};
 
 
 
void mouseHandler(int event, int x, int y, int flags, void* img)
 
{
 
 switch(event){
 
 case CV_EVENT_LBUTTONDOWN:
 
 // system("cls"); // Uncomment to clear screen.
 
 cerr  
 RgbPixel pxRgb = getRgbPixel(x,y, (Mat*)img);
 
 cerr  
 break;
 
 }
 
}
 
 
 
void segPickHandler(int event, int x, int y, int flags, void* iss)
 
{
 
 ImgSegSet* issTemp = (ImgSegSet*)iss;
 
 
 
 /// Mark a segment on screen and print its properties (id, features, ...) on standard output. ///
 
 if ( event == CV_EVENT_LBUTTONDOWN )
 
 {
 
 cerr  
 // ...
 
 
 
 
 
 
 
 }
 
 
 
 /// Assign group members. ///
 
 if ( event == CV_EVENT_LBUTTONUP )
 
 {
 
 if(flags & CV_EVENT_FLAG_CTRLKEY)
 
 {
 
 cerr  
 
 
 /// Assign segment to group A. ///
 
 // ...
 
 }
 
 
 
 if(flags & CV_EVENT_FLAG_SHIFTKEY)
 
 {
 
 cerr  
 
 
 /// Assign segment to group B. ///
 
 // ...
 
 }
 
 
 
 }
 
 
 
 
 
 /// Apply classifier. ///
 
 if ( event == CV_EVENT_RBUTTONUP )
 
 {
 
 cerr  
 
 
 /// Fill training and test set and apply classifier. ///
 
 // ...
 
 }
 
 
 
}
 
 
 
/*** MAIN FUNCTION ***/
 
 
 
int main(int argc, char* argv[])
 
{
 
 // Empty image pointer for all images.
 
 Mat* img_color = 0;
 
 Mat* img_segments = 0;
 
 
 
 // Load image from file.
 
 if ( argc != 2 ) { cerr  
 img_color = new Mat(imread( argv[1], CV_LOAD_IMAGE_UNCHANGED ));
 
 // img_color = cvLoadImage("Images/ErdCash001-resized.jpg");
 
 img_segments = new Mat(img_color->clone());
 
 
 
 //run through image (bild wird dadurch grn)
 
 for(int i = 0; i cols; i++)
 
 {
 
 for(int j = 0; j rows; j++)
 
 {
 
 //get pixel value
 
 RgbPixel pix= getRgbPixel(i, j, img_color);
 
 //cout  
RgbPixel pixNew = pix;
 
 //set blue and red to zero
 
 pixNew.b = 0;
 
 //pixNew.g = 0;
 
 pixNew.r = 0;
 
 //set pixel
 
 setRgbPixel(i, j, pixNew, img_segments );
 
 }
 
 }
 
 
 
 if( !img_color ){ cerr 
 
 printf("Processing a %dx%d image with %d channels ",img_color->cols,img_color->rows,img_color->channels());
 
 
 
 // Create some windows.
 
 namedWindow("original image", CV_WINDOW_AUTOSIZE);
 
 cvMoveWindow("original image", 50, 30);
 
 namedWindow("segmented objects", CV_WINDOW_AUTOSIZE);
 
 cvMoveWindow("segmented objects", 700, 30);
 
 
 
 
 
 // Create vector of segments.
 
 vector segVec;
 
 
 
 
 
 /// Define background and segment colors. ///
 
 RgbPixel backgroundPx2;
 
 backgroundPx2.r=0;
 
 backgroundPx2.g=0;
 
 backgroundPx2.b=0;
 
 setRgbPixel(0,0,backgroundPx2,img_segments);
 
 
 
 /// Do flood fill segmentation of background using OpenCV cvFloodFill(). ///
 
floodFill(*img_segments, Point(255,255), Scalar(0,0,0), 0, Scalar(100,100,100), Scalar(50,50,50), FLOODFILL_FIXED_RANGE);
 
 
 
 /// Binarise image after background segmentation. ///
 
threshold(*img_segments, *img_segments, 0, 255, THRESH_BINARY);
 
 
 
 
 
 /// Do segmentation of every object in image. ///
 
erode(*img_segments, *img_segments, Mat(), Point(-1,-1), 1, 1, 1);
 
dilate(*img_segments, *img_segments, Mat(), Point(-1,-1), 1, 1, 1);
 
 
 
 
 
 
 
 /// Create a segment object for every segmented image area and add it to vector of segments. ///
 
 // ...
 
 
 
 /// Attach all pixels to the corresponding segments. ///
 
 // ...
 
 
 
 /// Delete all segments which are too small (not enough pixels). ///
 
 // ...
 
 
 
 
 
 // Show updated image in window.
 
 imshow( "segmented objects", *img_segments );
 
imshow( "original image", *img_color );
 
 
 
 
 
 // Set mouse handlers for both windows.
 
 setMouseCallback( "original image", mouseHandler, img_color );
 
 
 
 ImgSegSet iss;
 
 iss.img = img_segments;
 
 iss.segVec = &segVec;
 
 setMouseCallback( "segmented objects", segPickHandler, &iss );
 
 
 
 // Wait for any key.
 
 while ( true )
 
 {
 
 waitKey(0);
 
 }
 
 
 
 /// Clean up of vector of segments. ///
 
 // ...
 
 
 
 // Release the images.
 
 //releaseImage( &img_color );
 
 //cvReleaseImage( &img_segments );
 
 
 
 // Return with 0.
 
 
 
 
 
 return 0;
 
}