Skip to content
Snippets Groups Projects
Commit e6b37802 authored by Otmane Lahlou's avatar Otmane Lahlou
Browse files

ENH : enhance otbTestDriver to use the otbTestMain (Julien made)

parent e64c3e41
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Code/Testing/otbTestDriver.cxx
+ 62
− 390
View file @ e6b37802
......@@ -28,34 +28,24 @@
#include <string>
#include <iostream>
#include <fstream>
#include "itkNumericTraits.h"
#include "itkMultiThreader.h"
#include "otbImage.h"
#include "otbImageFileReader.h"
#include "otbImageFileWriter.h"
#include "itkImageRegionConstIterator.h"
#include "itkSubtractImageFilter.h"
#include "itkRescaleIntensityImageFilter.h"
#include "itkExtractImageFilter.h"
#include "itkDifferenceImageFilter.h"
#include "itkImageRegion.h"
#include "itksys/SystemTools.hxx"
// include SharedForward to avoid duplicating the code which find the library path variable
// name and the path separator
#include "itksys/SharedForward.h"
#include "itksys/Process.h"
#include "otbTestHelper.h"
#define ITK_TEST_DIMENSION_MAX 6
// Temporary definition of otbTestMain
int otbTestMain(int arc, char * arv[]);
/** Display usage */
void usage()
{
std::cerr << "usage: otbTestDriver [options] prg [args]" << std::endl;
std::cerr << "usage: otbTestDriver [global_options] [non_regression_commands] Execute prg [args]" << std::endl;
std::cerr << std::endl;
std::cerr << "otbTestDriver alter the environment, run a test program and compare the images" << std::endl;
std::cerr << "produced." << std::endl;
std::cerr << "otbTestDriver alter the environment, run a test program and does regression testing based on capabilities provided by otbTestMain.h" << std::endl;
std::cerr << std::endl;
std::cerr << "Options:" << std::endl;
std::cerr << "Global pptions:" << std::endl;
std::cerr << " --add-before-libpath PATH" << std::endl;
std::cerr << " Add a path to the library path environment. This option take care of" << std::endl;
std::cerr << " choosing the right environment variable for your system." << std::endl;
......@@ -65,276 +55,17 @@ void usage()
std::cerr << " Add a VALUE to the variable name in the environment." << std::endl;
std::cerr << " This option can be used several times." << std::endl;
std::cerr << std::endl;
std::cerr << " --compare TEST BASELINE" << std::endl;
std::cerr << " Compare the TEST image to the BASELINE one." << std::endl;
std::cerr << " This option can be used several times." << std::endl;
std::cerr << std::endl;
std::cerr << " --" << std::endl;
std::cerr << " The options after -- are not interpreted by this program and passed" << std::endl;
std::cerr << " directly to the test program." << std::endl;
std::cerr << std::endl;
std::cerr << " --help" << std::endl;
std::cerr << " Display this message and exit." << std::endl;
std::cerr << std::endl;
}
// Regression Testing Code
int RegressionTestImage (const char *testImageFilename, const char *baselineImageFilename, int reportErrors)
/** This function parses the command line and process everything
* related to the --add-before-libpath, --add-before-env and --help.
* Every other args are added to the remainingArgs vector */
int parseCommandLine(int ac, char * av[], std::vector<char *> & remainingArgs)
{
// Use the factory mechanism to read the test and baseline files and convert them to double
typedef otb::Image<double,ITK_TEST_DIMENSION_MAX> ImageType;
typedef otb::Image<unsigned char,ITK_TEST_DIMENSION_MAX> OutputType;
typedef otb::Image<unsigned char,2> DiffOutputType;
typedef otb::ImageFileReader<ImageType> ReaderType;
// Read the baseline file
ReaderType::Pointer baselineReader = ReaderType::New();
baselineReader->SetFileName(baselineImageFilename);
try
{
baselineReader->UpdateLargestPossibleRegion();
}
catch (itk::ExceptionObject& e)
{
std::cerr << "Exception detected while reading " << baselineImageFilename << " : " << e.GetDescription();
return 1000;
}
// Read the file generated by the test
ReaderType::Pointer testReader = ReaderType::New();
testReader->SetFileName(testImageFilename);
try
{
testReader->UpdateLargestPossibleRegion();
}
catch (itk::ExceptionObject& e)
{
std::cerr << "Exception detected while reading " << testImageFilename << " : " << e.GetDescription() << std::endl;
return 1000;
}
// The sizes of the baseline and test image must match
ImageType::SizeType baselineSize;
baselineSize = baselineReader->GetOutput()->GetLargestPossibleRegion().GetSize();
ImageType::SizeType testSize;
testSize = testReader->GetOutput()->GetLargestPossibleRegion().GetSize();
if (baselineSize != testSize)
{
std::cerr << "The size of the Baseline image and Test image do not match!" << std::endl;
std::cerr << "Baseline image: " << baselineImageFilename
<< " has size " << baselineSize << std::endl;
std::cerr << "Test image: " << testImageFilename
<< " has size " << testSize << std::endl;
return 1;
}
// Now compare the two images
typedef itk::DifferenceImageFilter<ImageType,ImageType> DiffType;
DiffType::Pointer diff = DiffType::New();
diff->SetValidInput(baselineReader->GetOutput());
diff->SetTestInput(testReader->GetOutput());
diff->SetDifferenceThreshold(2.0);
diff->UpdateLargestPossibleRegion();
double status = diff->GetTotalDifference();
// if there are discrepencies, create an diff image
if (status && reportErrors)
{
typedef itk::RescaleIntensityImageFilter<ImageType,OutputType> RescaleType;
typedef itk::ExtractImageFilter<OutputType,DiffOutputType> ExtractType;
typedef otb::ImageFileWriter<DiffOutputType> WriterType;
typedef itk::ImageRegion<ITK_TEST_DIMENSION_MAX> RegionType;
OutputType::IndexType index; index.Fill(0);
OutputType::SizeType size; size.Fill(0);
RescaleType::Pointer rescale = RescaleType::New();
rescale->SetOutputMinimum(itk::NumericTraits<unsigned char>::NonpositiveMin());
rescale->SetOutputMaximum(itk::NumericTraits<unsigned char>::max());
rescale->SetInput(diff->GetOutput());
rescale->UpdateLargestPossibleRegion();
RegionType region;
region.SetIndex(index);
size = rescale->GetOutput()->GetLargestPossibleRegion().GetSize();
for (unsigned int i = 2; i < ITK_TEST_DIMENSION_MAX; i++)
{
size[i] = 0;
}
region.SetSize(size);
ExtractType::Pointer extract = ExtractType::New();
extract->SetInput(rescale->GetOutput());
extract->SetExtractionRegion(region);
WriterType::Pointer writer = WriterType::New();
writer->SetInput(extract->GetOutput());
std::cout << "<DartMeasurement name=\"ImageError\" type=\"numeric/double\">";
std::cout << status;
std::cout << "</DartMeasurement>" << std::endl;
::itk::OStringStream diffName;
diffName << testImageFilename << ".diff.png";
try
{
rescale->SetInput(diff->GetOutput());
rescale->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during rescale of " << diffName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during rescale of " << diffName.str() << std::endl;
}
writer->SetFileName(diffName.str().c_str());
try
{
writer->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during write of " << diffName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during write of " << diffName.str() << std::endl;
}
std::cout << "<DartMeasurementFile name=\"DifferenceImage\" type=\"image/png\">";
std::cout << diffName.str();
std::cout << "</DartMeasurementFile>" << std::endl;
::itk::OStringStream baseName;
baseName << testImageFilename << ".base.png";
try
{
rescale->SetInput(baselineReader->GetOutput());
rescale->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during rescale of " << baseName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during rescale of " << baseName.str() << std::endl;
}
try
{
writer->SetFileName(baseName.str().c_str());
writer->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during write of " << baseName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during write of " << baseName.str() << std::endl;
}
std::cout << "<DartMeasurementFile name=\"BaselineImage\" type=\"image/png\">";
std::cout << baseName.str();
std::cout << "</DartMeasurementFile>" << std::endl;
::itk::OStringStream testName;
testName << testImageFilename << ".test.png";
try
{
rescale->SetInput(testReader->GetOutput());
rescale->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during rescale of " << testName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during rescale of " << testName.str() << std::endl;
}
try
{
writer->SetFileName(testName.str().c_str());
writer->Update();
}
catch(const std::exception& e)
{
std::cerr << "Error during write of " << testName.str() << std::endl;
std::cerr << e.what() << "\n";
}
catch (...)
{
std::cerr << "Error during write of " << testName.str() << std::endl;
}
std::cout << "<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
std::cout << testName.str();
std::cout << "</DartMeasurementFile>" << std::endl;
}
return (status != 0) ? 1 : 0;
}
//
// Generate all of the possible baselines
// The possible baselines are generated fromn the baselineFilename using the following algorithm:
// 1) strip the suffix
// 2) append a digit .x
// 3) append the original suffix.
// It the file exists, increment x and continue
//
std::map<std::string,int> RegressionTestBaselines (char *baselineFilename)
{
std::map<std::string,int> baselines;
baselines[std::string(baselineFilename)] = 0;
std::string originalBaseline(baselineFilename);
int x = 0;
std::string::size_type suffixPos = originalBaseline.rfind(".");
std::string suffix;
if (suffixPos != std::string::npos)
{
suffix = originalBaseline.substr(suffixPos,originalBaseline.length());
originalBaseline.erase(suffixPos,originalBaseline.length());
}
while (++x)
{
::itk::OStringStream filename;
filename << originalBaseline << "." << x << suffix;
std::ifstream filestream(filename.str().c_str());
if (!filestream)
{
break;
}
baselines[filename.str()] = 0;
filestream.close();
}
return baselines;
}
int main(int ac, char* av[] )
{
std::vector< char* > args;
typedef std::pair< char *, char *> ComparePairType;
std::vector< ComparePairType > compareList;
otb::TestHelper testHelper;
// parse the command line
int i = 1;
bool skip = false;
......@@ -395,21 +126,6 @@ int main(int ac, char* av[] )
itksys::SystemTools::PutEnv( env.c_str() );
i += 3;
}
else if( !skip && strcmp(av[i], "--compare") == 0 )
{
if( i+2 >= ac )
{
usage();
return 1;
}
compareList.push_back( ComparePairType( av[i+1], av[i+2] ) );
i += 3;
}
else if( !skip && strcmp(av[i], "--") == 0 )
{
skip = true;
i += 1;
}
else if( !skip && strcmp(av[i], "--help") == 0 )
{
usage();
......@@ -417,117 +133,73 @@ int main(int ac, char* av[] )
}
else
{
args.push_back( av[i] );
remainingArgs.push_back( av[i] );
i += 1;
}
}
return 0;
}
if( args.empty() )
int main(int ac, char* av[] )
{
// A vector to store remaining args
std::vector< char* > remainingArgs;
// First parse the command line for system wide options
int ret = parseCommandLine(ac,av,remainingArgs);
// Check for the return code
if(ret)
{
usage();
std::cerr<<"Error while parsing arguments, exiting ..."<<std::endl;
return 1;
}
// a NULL is required at the end of the table
char** argv = new char*[ args.size() + 1 ];
for( i=0; i<static_cast<int>(args.size()); i++ )
// Check if there are remaining args
if( remainingArgs.empty() )
{
argv[ i ] = args[ i ];
}
argv[ args.size() ] = NULL;
usage();
return 1;
}
// a NULL is required at the end of the table
char** argv = new char*[ remainingArgs.size() + 2 ];
argv[0] = "dummy";
for(int i=0; i<static_cast<int>(remainingArgs.size()); i++ )
{
argv[ i+1 ] = remainingArgs[ i ];
}
argv[ remainingArgs.size() + 1 ] = NULL;
/** Call to the otbTestMain */
return otbTestMain(remainingArgs.size(),argv);
}
// This is a dummy main to be registered as a test for the otbTestMain
int Execute(int argc, char * argv[])
{
argc-=1;
argv+=1;
// Create the appropriate itk process
itksysProcess * process = itksysProcess_New();
itksysProcess_SetCommand( process, argv );
itksysProcess_SetPipeShared( process, itksysProcess_Pipe_STDOUT, true);
itksysProcess_SetPipeShared( process, itksysProcess_Pipe_STDERR, true);
itksysProcess_Execute( process );
itksysProcess_WaitForExit( process, NULL );
delete []argv;
int retCode = itksysProcess_GetExitValue( process );
if( retCode != 0 )
{
// no need to compare the images: the test has failed
return retCode;
}
itksysProcess_WaitForExit( process, NULL );
int retCode = itksysProcess_GetExitValue(process);;
return retCode;
}
int cpt(1);
// Include otbTestMain and switch otbTestMain with main definition in otbTestMain.h
#undef otbTestMain
#undef main
#define main otbTestMain
#include "otbTestMain.h"
// now compare the images
try
{
for( i=0; i<static_cast<int>(compareList.size()); i++)
{
char * testFilename = compareList[i].first;
char * baselineFilename = compareList[i].second;
std::cout << "testFilename: " << testFilename << " baselineFilename: " << baselineFilename << std::endl;
// Make a list of possible baselines
std::map<std::string,int> baselines = testHelper.RegressionTestbaselines(baselineFilename);
std::map<std::string,int>::iterator baseline = baselines.begin();
std::string bestBaseline;
int bestBaselineStatus = itk::NumericTraits<int>::max();
while (baseline != baselines.end())
{
baseline->second = testHelper.RegressionTestImage(cpt,
testFilename,
(baseline->first).c_str(),
0);
if (baseline->second < bestBaselineStatus)
{
bestBaseline = baseline->first;
bestBaselineStatus = baseline->second;
}
if (baseline->second == 0)
{
break;
}
++baseline;
}
// if the best we can do still has errors, generate the error images
if (bestBaselineStatus)
{
// baseline->second = RegressionTestImage(testFilename,
// bestBaseline.c_str(),
// 1);
void RegisterTests()
{
REGISTER_TEST(Execute);
}
baseline->second = testHelper.RegressionTestImage(cpt,
testFilename,
bestBaseline.c_str(),
0);
}
// output the matching baseline
std::cout << "<DartMeasurement name=\"BaselineImageName\" type=\"text/string\">";
std::cout << itksys::SystemTools::GetFilenameName(bestBaseline);
std::cout << "</DartMeasurement>" << std::endl;
if( bestBaselineStatus != 0 )
{
return bestBaselineStatus;
}
}
}
catch(const itk::ExceptionObject& e)
{
std::cerr << "ITK test driver caught an ITK exception:\n";
std::cerr << e.GetFile() << ":" << e.GetLine() << ":\n"
<< e.GetDescription() << "\n";
return -1;
}
catch(const std::exception& e)
{
std::cerr << "ITK test driver caught an exception:\n";
std::cerr << e.what() << "\n";
return -1;
}
catch(...)
{
std::cerr << "ITK test driver caught an unknown exception!!!\n";
return -1;
}
return 0;
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment