lti::classificationStatistics Class Reference

This class allows the generation of classification statistics. More...

#include <ltiClassificationStatistics.h>

Inheritance diagram for lti::classificationStatistics:

[legend]

Collaboration diagram for lti::classificationStatistics:

[legend]

List of all members.

Classes
class	parameters
	the parameters for the class classificationStatistics More...
Public Types
enum	eStatistics { CorrectSingle = -1, CorrectMultiple = -2, CorrectReject = -3, WrongSingle = -4, WrongMultiple = -5, WrongReject = -6, Error = -7 }
Public Member Functions
	classificationStatistics ()
	classificationStatistics (const parameters &par)
	~classificationStatistics ()
bool	setParameters (const parameters &params)
const parameters &	getParameters () const
eStatistics	consider (const classifier::outputVector &result, const int &realObjectID, const std::string &patternInfo="-?-")
void	reset ()
void	flush ()
bool	writeData (std::ostream &ostr)
bool	setNames (const std::map< int, std::string > &idMap)
Data access members
Use these to access the internal data members
const bool &	getSaved () const
const dynamicMatrix< double > &	getConfusionMatrix () const
const std::map< int, std::string > &	getIdToNamesMap () const
Statistics access members
Use these to access the accumulated statistics
const int &	getNumberOfResults () const
const int &	getCorrectSingle () const
const int &	getCorrectMultiple () const
const int &	getCorrectReject () const
const int &	getWrongSingle () const
const int &	getWrongMultiple () const
const int &	getWrongReject () const
const dynamicMatrix< double > &	getNBestMatrix () const
const dynamicMatrix< double > &	getNBestMultipleMatrix () const
const dynamicMatrix< double > &	getMinPlacesMatrix () const
Static Public Member Functions
static bool	loadNames (const std::string &filename, std::map< int, std::string > &idToNamesMap, const bool namesInFileFirst=false)
Protected Member Functions
bool	writeData ()
bool	writeConfusionMatrix (std::ostream &out, const dynamicMatrix< double > &matrix)
bool	writeNBestMatrix (std::ostream &out, const dynamicMatrix< double > &matrix, const std::string &title, const bool &normalize, const std::string &labelSingular="Best", const std::string &labelPlural="Best")
bool	writeLogFileHeader (const std::string &filename)
bool	writeLogFile (const classifier::outputVector &result, const eStatistics &rState, const int &realObjectID, const std::string &patternInfo)
void	sortObjNames (const dynamicMatrix< double > &matrix, std::vector< int > &seqToRealId)
void	format (std::ostream &out, const int &width, const double &val) const
void	format (std::ostream &out, const int &width, const int &val) const
bool	inscribeObject (const std::string &name, const int &id)
Protected Attributes
parameters	param
bool	saved
dynamicMatrix< double >	confMatrix
std::map< int, std::string >	idToNamesMap
Statistics
These are the internal members for storing the actual classification statistics
int	numberOfResults
int	correctSingle
int	correctMultiple
int	correctReject
int	wrongSingle
int	wrongMultiple
int	wrongReject
dynamicMatrix< double >	nBest
dynamicMatrix< double >	nBestMultiple
dynamicMatrix< double >	minPlaces
Static Protected Attributes
static const int	logWidths []
static const int	confWidths []

Detailed Description

This class allows the generation of classification statistics.

This object will generate some confusion matrices and log-files, which can help to the selection of classifier-parameters.

The results of the classification will be stored directly into the user specified files.

Following example shows how to use this object in the statistics for classification results.

 lti::rbf theNet;     // a rbf-Network

 // Training data set:
 // Some points in a two-dimensional feature-space will belong to one
 // of two classes: circle or square
 lti::dmatrix train(10,2);
 double trainData[20] = { 0, 3, // circle
                          1, 3, // circle
                          2, 3, // circle
                          0, 2, // circle
                          1, 2, // circle
                          2, 1, // square
                          3, 1, // square
                          4, 1, // square
                          3, 0, // square
                          4, 0};// square
 train.fill(trainData);

 // the ids for the patterns in the training set
 lti::ivector trainIds(10);
 int trainClasses[10] = {314,314,314,314,314,4,4,4,4,4};
 trainIds.fill(trainClasses);
 
 // just to see what's happening when training...
 lti::streamProgressInfo progBox;
 theNet.setProgressObject(progBox);
 
 // train the classifier
 theNet.train(train,trainIds);
    
 // ------------- STATISTICS --------------
 
 // prepare the statistics object
 lti::rbf::outputVector nnoutput; // output of the rbf network
 lti::classificationStatistics statistics;
 lti::classificationStatistics::parameters statParam;
 
 // all statistic files should be stored in the "tmp" directory
 statParam.path = "/tmp/";
 
 // and of course, indicate the statistics-object which parameters
 // should be used:
 statistics.setParameters(statParam);

 // assign some human-readable names to the object-Ids
 std::map<int,std::string> names;
 names[314] = "Circle";
 names[4]   = "Square";
 
 statistics.setNames(names);
 
 // now test some points of the 2D-feature space, and generate
 // some statistics about the classification:
 
 double x,y; // the coordinates of a point in the feature space
 int realID; // for the statistics is required the real id of
 // the given feature, to check if the classification is
 // correct or not!
 
 char buffer[1024]; // a temporary string buffer
 
 lti::dvector feature(2);// a feature-vector

 for (y=0;y<=4.0;y+=0.5) {
   for (x=0;x<=4.0;x+=0.5) {
     realID = (y>x) ? 314 : 4; // generate the real id!
  
     feature.at(0)=x;          // initialize the feature vector
     feature.at(1)=y;

     theNet.classify(feature,nnoutput);// classify the vector!

     // generate a feature-name for the feature
     sprintf(buffer,"x: %.2f, y: %.2f",x,y);

     // consider the actual feature in the statistics
     statistics.consider(nnoutput,realID,buffer);
   }
 }

 statistics.flush(); // save all the statistics on disk!

Two files will be created: /tmp/logfile.txt with the list of the wrong classifications and /tmp/confusion.mat with the confusion matrices.

The confusion matrix file has following information:

How many classes were considered in the classification.
The confusion matrix itself, which show at each row how many percent of the presentations of the given object were recognized as the given objects. For example, the previous code will produce following matrix:

 ---     Circle  Square correctSingle correctMultip correctReject ...
 Circle   1.000   0.000      1.000000      0.000000      0.000000 ...
 Square   0.222   0.778      0.777778      0.000000      0.000000 ...

which means that the circle were allways correctly recognized, and the square was 22% wrong and 78% correctly classified.

Total statistics. Here will be listed how many classifications were considered in the statistics, the correct and wrong classifications and the rejections.
The n-Best statistic indicates that the correct object was at the n-th position
The shared n-Best position indicates how many of the classifications were shared between different classes.
The mininal number of positions indicates how many positions need to be taken into consideration to assure a correct classification.

Member Enumeration Documentation

enum lti::classificationStatistics::eStatistics

types of classification results

Enumerator:

CorrectSingle	classifier chooses one class and it is the correct one
CorrectMultiple	classifier chooses many classes and one of them is correct
CorrectReject	classifier rejects one class which is indeed not the winner
WrongSingle	classifier chooses one class and it is wrong
WrongMultiple	classifier chooses many classes and all of theme are wrong
WrongReject	classifier rejects a class which would have been the correct winner
Error	there is an error on the data

Constructor & Destructor Documentation

lti::classificationStatistics::classificationStatistics ( )

default constructor

lti::classificationStatistics::classificationStatistics ( const parameters & par )

default constructor with parameters

lti::classificationStatistics::~classificationStatistics ( )

Destructor.

Member Function Documentation

eStatistics lti::classificationStatistics::consider	(	const classifier::outputVector &	result,
		const int &	realObjectID,
		const std::string &	patternInfo = `"-?-"`
	)

include a new classification result into the statistics.

The patternInfo is additional text that will appear in the logFile. Returns the recognition state (see eStatistics).

Note: the ID must be positive. negative indices are discarded.

void lti::classificationStatistics::flush ( )

save all statistics stored by now into the file specified in the parameters.

Note that the parameter writeConfusionMatrix must be set to true.

void lti::classificationStatistics::format	(	std::ostream &	out,
		const int &	width,
		const int &	val
	)			const `[protected]`

workaround for std::ios bug in gcc when writing double numbers

void lti::classificationStatistics::format	(	std::ostream &	out,
		const int &	width,
		const double &	val
	)			const `[protected]`

workaround for std::ios bug in gcc when writing double numbers

const dynamicMatrix<double>& lti::classificationStatistics::getConfusionMatrix ( ) const

returns a const reference to the confusion matrix

const int& lti::classificationStatistics::getCorrectMultiple ( ) const

number of correct multiples

const int& lti::classificationStatistics::getCorrectReject ( ) const

number of correct rejects

const int& lti::classificationStatistics::getCorrectSingle ( ) const

number of correct singles

const std::map<int,std::string>& lti::classificationStatistics::getIdToNamesMap ( ) const

returns a const reference to the idToNamesMap

const dynamicMatrix<double>& lti::classificationStatistics::getMinPlacesMatrix ( ) const

minPlaces, i.e.

minimum number of elements required in order to contain the proper class.

const dynamicMatrix<double>& lti::classificationStatistics::getNBestMatrix ( ) const

nBest recognition results for single correct

const dynamicMatrix<double>& lti::classificationStatistics::getNBestMultipleMatrix ( ) const

nBest recognition results for multiple correct

const int& lti::classificationStatistics::getNumberOfResults ( ) const

returns a const reference to the number of results

const parameters& lti::classificationStatistics::getParameters ( ) const

returns a const reference to the parameters

const bool& lti::classificationStatistics::getSaved ( ) const

returns a const reference to protected member "saved"

const int& lti::classificationStatistics::getWrongMultiple ( ) const

number of wrong multiples

const int& lti::classificationStatistics::getWrongReject ( ) const

number of wrong rejects

const int& lti::classificationStatistics::getWrongSingle ( ) const

number of wrong singles

bool lti::classificationStatistics::inscribeObject	(	const std::string &	name,
		const int &	id
	)			`[protected]`

use in the statistics an object with the given name and real id number

static bool lti::classificationStatistics::loadNames	(	const std::string &	filename,
		std::map< int, std::string > &	idToNamesMap,
		const bool	namesInFileFirst = `false`
	)			`[static]`

load the file "filename", which contains the correspondences between a class id number and a class name.

Note: calling this does NOT set the internal idToNamesMap!!!

The file must consist of two columns. The first containing the object ids, the second the object names (optionally in double quotes) and separated by whitespaces e.g.

 314 "Circle"
 4   "Square"

Parameters:

	filename	the name of the file to be used
	idToNamesMap	the map where the correspondences will be written. all previous data in the map will be kept.
	namesInFileFirst	invert the expected order of the data.

void lti::classificationStatistics::reset ( )

reset clears the statistics

bool lti::classificationStatistics::setNames ( const std::map< int, std::string > & idMap )

set the correspondences between a class id number and a class name.

Parameters:

idMap

the map where the correspondences are given.

bool lti::classificationStatistics::setParameters ( const parameters & params )

set the parameters

void lti::classificationStatistics::sortObjNames	(	const dynamicMatrix< double > &	matrix,
		std::vector< int > &	seqToRealId
	)			`[protected]`

sort the object names in the matrices

bool lti::classificationStatistics::writeConfusionMatrix	(	std::ostream &	out,
		const dynamicMatrix< double > &	matrix
	)			`[protected]`

write confusion matrix to file

bool lti::classificationStatistics::writeData ( ) [protected]

save all statistic data in the respective files

bool lti::classificationStatistics::writeData ( std::ostream & ostr )

write all accumulated statistic data to the specified stream.

this can be used to manually save the statistics.

bool lti::classificationStatistics::writeLogFile	(	const classifier::outputVector &	result,
		const eStatistics &	rState,
		const int &	realObjectID,
		const std::string &	patternInfo
	)			`[protected]`

write classification to file

bool lti::classificationStatistics::writeLogFileHeader ( const std::string & filename ) [protected]

write classification to file

bool lti::classificationStatistics::writeNBestMatrix	(	std::ostream &	out,
		const dynamicMatrix< double > &	matrix,
		const std::string &	title,
		const bool &	normalize,
		const std::string &	labelSingular = `"Best"`,
		const std::string &	labelPlural = `"Best"`
	)			`[protected]`

write nBest-matrix

Member Data Documentation

dynamicMatrix<double> lti::classificationStatistics::confMatrix [protected]

the confusion matrix stores the classification results, each row represents one class (row index = realObjectID) and the classification results for this class

const int lti::classificationStatistics::confWidths[] [static, protected]

formatting widths for the columns of the confusion matrix file

int lti::classificationStatistics::correctMultiple [protected]

number of correct multiples

int lti::classificationStatistics::correctReject [protected]

number of correct rejects

int lti::classificationStatistics::correctSingle [protected]

number of correct singles

std::map<int,std::string> lti::classificationStatistics::idToNamesMap [protected]

map an id number to the name of the object.

const int lti::classificationStatistics::logWidths[] [static, protected]

formatting widths for the columns of the log file