ltiSegmentationEvaluation.h

/*
 * Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003
 * Lehrstuhl fuer Technische Informatik, RWTH-Aachen, Germany
 *
 * This file is part of the LTI-Computer Vision Library (LTI-Lib)
 *
 * The LTI-Lib is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License (LGPL)
 * as published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * The LTI-Lib is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with the LTI-Lib; see the file LICENSE.  If
 * not, write to the Free Software Foundation, Inc., 59 Temple Place -
 * Suite 330, Boston, MA 02111-1307, USA.
 */


/*--------------------------------------------------------------------
 * project ....: LTI-Lib: Image Processing and Computer Vision Library
 * file .......: ltiSegmentationEvaluation.h
 * authors ....: Pablo Alvarado
 * organization: LTI, RWTH Aachen
 * creation ...: 25.11.2003
 * revisions ..: $Id: ltiSegmentationEvaluation.h,v 1.1 2006/03/10 02:53:11 alvarado Exp $
 */

#ifndef _LTI_SEGMENTATION_EVALUATION_TESTER_H_
#define _LTI_SEGMENTATION_EVALUATION_TESTER_H_

#include "ltiParetoFront.h"
#include "ltiImage.h"

namespace lti {

  /**
   * Pareto evaluation of segmentation functors.
   *
   * This abstract class provides a basic framework for the evaluation of
   * segmentation algorithms.  The functor evaluates a result comparing it
   * with some ground truth data, but it does not know anything about the
   * parameterization of any algorithm, which should be done by derived
   * classes.
   *
   * Several measures are provided to evaluate the algorithms:
   * - numberOfRegions
   * - numRegionsReciprocal (1/number of regions)
   * - pixelWisePotentialAccuracy
   * - objectWisePotentialAccuracy
   * - regionWiseInformationContent
   * - throughput
   * - regionIntegrity
   * - pixelWiseCertainty
   *
   * For the evaluation of a parameterization the algorithm takes all
   * images stored in the file specified in parameters::images.
   *
   * Following five methods have to be reimplemented in derived classes:
   *
   * For the pareto front interface
   * - chromosomeToPhenotype()
   * - phenotypeToChromosome()
   * - getChromosomeSize()
   *
   * For the real segmentation task
   * - init()
   * - segment() 
   * - getInstanceOfParam()
   *
   * In the parameters you should define two sets of values for minimum and
   * maximum tolerable values (usually called minValues and maxValues).  The
   * chromosomeToPhenotype method has to take care that the values produced 
   * lie in the corresponding intervals.
   *
   * The definition of the used fitness measures can be found in the paper:
   *
   * Mark Everingham, Henk Muller and Barry Thomas, 
   * "Evaluating Image Segmentation Algoritnms using the Pareto Front".
   * Proceedings of the 7th European Conference on Computer Vision,
   * 2002.
   *
   * except the region integrity and pixel-wise certainty, which are described
   * in
   *
   * Pablo Alvarado, "Segmentation of color images for interactive 3D 
   *                  object retrieval", Dissertation, Aachen, 2004.
   */
  class segmentationEvaluation : public paretoFront {
  public:

    /**
     * The parameters for the class segmentationEvaluation
     */
    class parameters : public paretoFront::parameters {
    public:
      /**
       * Default constructor
       */
      parameters();

      /**
       * Copy constructor
       * @param other the parameters object to be copied
       */
      parameters(const parameters& other);

      /**
       * Destructor
       */
      ~parameters();

      /**
       * Returns name of this type
       */
      const char* getTypeName() const;

      /**
       * Copy the contents of a parameters object
       * @param other the parameters object to be copied
       * @return a reference to this parameters object
       */
      parameters& copy(const parameters& other);

      /**
       * Copy the contents of a parameters object
       * @param other the parameters object to be copied
       * @return a reference to this parameters object
       */
      parameters& operator=(const parameters& other);


      /**
       * Returns a pointer to a clone of the parameters
       */
      virtual functor::parameters* clone() const;

      /**
       * Write the parameters in the given ioHandler
       * @param handler the ioHandler to be used
       * @param complete if true (the default) the enclosing begin/end will
       *        be also written, otherwise only the data block will be written.
       * @return true if write was successful
       */
      virtual bool write(ioHandler& handler,const bool& complete=true) const;

      /**
       * Read the parameters from the given ioHandler
       * @param handler the ioHandler to be used
       * @param complete if true (the default) the enclosing begin/end will
       *        be also written, otherwise only the data block will be written.
       * @return true if write was successful
       */
      virtual bool read(ioHandler& handler,const bool& complete=true);

#     ifdef _LTI_MSC_6
      /**
       * This function is required by MSVC only, as a workaround for a
       * very awful bug, which exists since MSVC V.4.0, and still by
       * V.6.0 with all bugfixes (so called "service packs") remains
       * there...  This method is also public due to another bug, so please
       * NEVER EVER call this method directly: use read() instead
       */
      bool readMS(ioHandler& handler,const bool& complete=true);

      /**
       * This function is required by MSVC only, as a workaround for a
       * very awful bug, which exists since MSVC V.4.0, and still by
       * V.6.0 with all bugfixes (so called "service packs") remains
       * there...  This method is also public due to another bug, so please
       * NEVER EVER call this method directly: use write() instead
       */
      bool writeMS(ioHandler& handler,const bool& complete=true) const;
#     endif

      // ------------------------------------------------
      // the parameters
      // ------------------------------------------------

      /**
       * @name Fitness measures
       *
       * Value containing the 1D fitness measures to be computed.
       * The constants defined here are:
       * - numberOfRegions
       * - numRegionsReciprocal (1/number of regions)
       * - pixelWisePotentialAccuracy
       * - objectWisePotentialAccuracy
       * - regionWiseInformationContent
       * - throughput (1/time)
       * - region integrity
       * - pixelWiseCertainty
       *
       * This sequence is important, since it determines the positions of
       * the single measures in the fitness output vector.
       *
       * If more fitness measures are selected than the inherited parameter
       * fitnessSpaceDimensionality, then the first ones in this sequence
       * will be selected.
       *
       * If less fitness measure are selected than the inherited parameter
       * fitnessSpaceDimensionality, then the rest of the vector will be
       * filled with zeros.
       */
      //@{
      /** 
       * Total number of regiong detected per image.
       *
       * This is usually a cost, and therefore you will never want to optimize
       * the number of regions as if it were a fitness, but, in some
       * applications, it may be necessary to search for the optimal
       * "over-segmentations", which will therefore use this as fitness.
       * 
       * Default value: false
       */
      bool numberOfRegions;

      /**
       * Reciprocal for the number of regions
       *
       * This is the inverse of the number of regions (1/number of regions).
       * Usually you want the get the less possible number of regions that
       * still don't merge different objects while partitioning an image.
       * 
       * This can be interpreted as the mean size of the regions normalized
       * to the size of the image.
       *
       * Default value: true
       */
      bool numRegionsReciprocal;

      /**
       * Pixel-wise potential accuracy.
       *
       * Describes the percentange of the image pixels than can be optimally
       * assigned to one of the ground-truth regions.
       *
       * This can be interpreted as a measure inversely proportional to the
       * degree of undersegmentation.
       *
       * Default value: true
       */
      bool pixelWisePotentialAccuracy;
      
      /**
       * Object-wise potential accuracy.
       *
       * Like pixel-wise potential accuracy but the contribution of each
       * reference region is normalized to its size so that all regions in
       * the reference segmentation are equally important.
       *
       * Default value: false
       */
      bool objectWisePotentialAccuracy;

      /**
       * Region-wise information content.
       *
       * Proportion of the reference region covered by each single region.
       *
       * Default value: false
       */
      bool regionWiseInformationContent;

      /**
       * Throughput
       *
       * Number of images segmented per second.
       *
       * Default value: false
       */
      bool throughput; 

      /**
       * Region integrity
       *
       * Measures the number of regions mapped to a reference region.  The
       * function used punishes over- and under-segmentation.
       * 
       * Default value: false
       */
      bool regionIntegrity;

      /**
       * Pixel-wise certainty
       *
       * This makes only sense for some segmentation functors, that can deliver
       * the certainty with which a pixel was assigned to a label.
       *
       * Default value: false
       */
      bool pixelWiseCertainty;
      //@}

      /**
       * Name of a file containing all images to be analyzed.
       *
       * Default value: "images.txt"
       */
      std::string images;

      /**
       * Mask postfix
       *
       * For each image file in \a images, a manual segmented mask is required
       * as "ground truth".
       *
       * The filename of the masks will be assumed to be the same
       * image name with the given postfix.  For example, if an image is
       * called "test.png" and the postfix is "-preseg", then the
       * mask will be assumed to be "test-preseg.png".
       *
       * Default value: "_premask"
       */
      std::string goldenPostfix;

      /**
       * Previous Mask Postfix.
       *
       * To save evaluation time, the test of higher-level algorithm can load
       * the lower-level segmentations of the images in a file with this
       * postfix and type lti (see lti::ltiFunctor), which is the only format
       * that can save imatrices.  For example, if an image is called
       * "test.png" and the prevStagePostfix is "-ibs", then the mask will
       * be assumed to be in "test-ibs.lti".
       *
       * If no file is found with these names, then a low-level segmentation 
       * will be assumed, which does not require any previous masks.
       *
       * Default value: "_ibs" 
       */
      std::string prevStagePostfix;

      /**
       * @name Fitness parameterizations
       */
      //@{
      /**
       * Minimal size for valid region
       *
       * Percentage of the image size that defines the minimal size of a
       * region to be considered in the statistics for region-wise information
       * content and object-wise potential accuracy.
       *
       * This value must be between 0 and 1.
       *
       * Default: 0.0005 (i.e. 0.05%)
       */
      float minValidRegionSize;
      //@}
    };

    /**
     * Default constructor
     */
    segmentationEvaluation();

    /**
     * Construct a functor using the given parameters
     */
    segmentationEvaluation(const parameters& par);

    /**
     * Copy constructor
     * @param other the object to be copied
     */
    segmentationEvaluation(const segmentationEvaluation& other);

    /**
     * Destructor
     */
    virtual ~segmentationEvaluation();

    /**
     * Returns the name of this type ("segmentationEvaluation")
     */
    virtual const char* getTypeName() const;

    /**
     * Returns used parameters
     */
    const parameters& getParameters() const;

    /**
     * Set the parameters
     */
    bool setParameters(const functor::parameters& other);

    /**
     * @name Public methods to be reimplemented
     */
    //@{

    /**
     * Convert a binary-chain representation of a chromosome to a valid
     * parameter object.
     */
    virtual bool chromosomeToPhenotype(const chromosome& genotype,
                                      functor::parameters& phenotype) const=0;

    /**
     * Return a fresh allocated parameters for the evaluated functor, which is
     * equivalent to the given genotype.
     */
    virtual functor::parameters* 
    chromosomeToPhenotype(const chromosome& genotype) const;

    /**
     * Evaluate Chromosome
     *
     * This method is one of the most important ones for the pareto evaluation.
     * Its task is to produce a multidimensional fitness measure for a given
     * chromosome.
     *
     * It returns true if the evaluation was successful, of false if the
     * phenotype represents some invalid parameterization.  It is highly 
     * recomended that the mutation and crossover methods are reimplemented to 
     * avoid invalid parameterizations.
     */
    virtual bool evaluateChromosome(const chromosome& individual,
                                    dvector& fitness); 

    //@}

    /**
     * @name Evaluation Functions
     *
     * The real evaluation is provided here in order to compute measures
     * independently of the Pareto Front.
     */
    //@{
    /**
     * Index names for the complete multidimensional fitness measure
     */
    enum {
      IdxNumRegions,        /**< Mean number of regions.                   */
      IdxNumRegionsRecip,   /**< Reciprocal of the mean number of regions. */
      IdxPixelAccuracy,     /**< Pixel wise potential accuracy.            */
      IdxObjectAccuracy,    /**< Object wise potential accuracy.           */
      IdxRegionWiseInfo,    /**< Region wise information content.          */
      IdxThroughput,        /**< 1.0/time in s^-1, meaning the number of
                             *   complete segmentations per second.
                             */
      IdxRegionIntegrity,   /**< Region integrity                          */
      IdxPixelCertainty     /**< Pixel-wise certainty                      */
    };

    /**
     * Evaluate the set of images (in the parameters) using the
     * given parameterization.
     *
     * @param param parameters for the segmentation functor to be used in the
     *              evaluation.
     * @param fitness multidimensional fitness measures for all images
     */
    bool evaluate(const functor::parameters& param,dvector& fitness);

    /**
     * Evaluation of the segmentation with the given channel using
     * the given parameterization.
     *
     * The real measures for the single image are obtained with the
     * element-wise division between fitness and norm.  But if several images
     * are used in the computations, the norms of each image have to be added
     * first.
     *
     * It is assumed that there is no previous stage (empty previous stage
     * mask).
     */
    virtual bool evaluate(const image& img,
                          const channel8& mask,
                          const functor::parameters& param,
                          dvector& fitness,
                          dvector& norm);

    /**
     * Evaluation of the segmentation with the given channel using
     * the given parameterization.
     *
     * The real measures for the single image are obtained with the
     * element-wise division between fitness and norm.  But if several images
     * are used in the computations, the norms of each image have to be added
     * first.
     */
    virtual bool evaluate(const image& img,
                          const imatrix& prevStage,
                          const channel8& mask,
                          const functor::parameters& param,
                          dvector& fitness,
                          dvector& norm);

    /**
     * Evaluation of the segmentation with the given channel using
     * a previously set parameterization.
     *
     * The real measures for the single image are obtained with the
     * element-wise division between fitness and norm.  But if several images
     * are used in the computations, the norms of each image have to be added
     * first.
     */
    virtual bool evaluate(const image& img,
                          const imatrix& prevStage,
                          const channel8& mask,
                          dvector& fitness,
                          dvector& norm);

    /**
     * Evaluation of segmentation
     *
     * Since you give already the results in this method, it is not possible to
     * compute the throughput and certainty measures.
     */
    virtual bool evaluate(const imatrix& result,
                          const channel8& refMask,
                          dvector& fitness,
                          dvector& norm,
                          const float minValidRegionSize=0.0005);
                          

    //@}

  protected:
    /**
     * Total number of fitness measures that will be computed
     * 
     */
    static const int totalFitnessDimensionality;

    /**
     * Cache for the images
     */
    std::vector<image> imageData;

    /**
     * Cache for the presegmentation masks
     */
    std::vector<channel8> maskData;

    /**
     * Cache for the previous stage masks
     */
    std::vector<imatrix> prevMaskData;

    /**
     * File names of the reference data
     */
    std::vector<std::string> goldenNames;

    /**
     * Read images and masks
     */
    bool readData();

    /**
     * Get the image name
     */
    std::string getMaskName(const std::string& imgName,
                            const std::string& postfix) const;

    /**
     * Get the image name of the previous stage file
     */
    std::string getPrevName(const std::string& imgName,
                            const std::string& postfix) const;

    /**
     * @name The segmentation evaluation methods
     */
    //@{
    /**
     * Initialize the segmentation functor
     */
    virtual bool init(const functor::parameters& param) = 0;

    /**
     * Segmentate an image using the parameterization set with init()
     *
     * @param img image to be segmented
     * @param prevStage segmentation mask from a previous stage.  Image-based
     *                  segmentation algorithms ignore this argument (in a
     *                  strict sense they consider a mask in which each pixel
     *                  has its own label).
     * @param mask new segmentation labeled mask.
     * @param certainty the certainty, with which the algorithm assigns each
     *                  pixel to its label.  If an algorithm does not provide
     *                  such measure (like almost all image-based algorithms)
     *                  this value will be empty, and for the computations
     *                  it will be assumed that all pixels have a certainty of
     *                  1.0 to be what they are.
     *
     * @return true if successful, false otherwise.
     */
    virtual bool segment(const image& img,
                         const imatrix& prevStage,
                         imatrix& mask,
                         channel& certainty) = 0;

    //@}

    /**
     * Return a new object with the correct parameters' type
     */
    virtual functor::parameters* getInstanceOfParam() const = 0;

    /**
     * Empty matrix used everywhere.
     */
    static const imatrix emptyMatrix;


  };
}

#endif