Commit aa5cd392 authored by Julien Malik's avatar Julien Malik

ENH: remove contribution of ExtractROI in memory print estimation

parent bb283629
......@@ -27,12 +27,11 @@
namespace otb
{
const double PipelineMemoryPrintCalculator::ByteToMegabyte = 1./vcl_pow(2.0, 20);
const double PipelineMemoryPrintCalculator::MegabyteToByte = vcl_pow(2.0, 20);
PipelineMemoryPrintCalculator
::PipelineMemoryPrintCalculator()
: m_MemoryPrint(0),
m_AvailableMemory(256000000),
m_OptimalNumberOfStreamDivisions(1),
m_DataToWrite(NULL),
m_BiasCorrectionFactor(1.),
m_VisitedProcessObjects()
......@@ -42,6 +41,17 @@ PipelineMemoryPrintCalculator
::~PipelineMemoryPrintCalculator()
{}
// [static]
unsigned long
PipelineMemoryPrintCalculator
::EstimateOptimalNumberOfStreamDivisions(MemoryPrintType memoryPrint, MemoryPrintType availableMemory)
{
unsigned long divisions;
divisions = vcl_ceil(static_cast<double>(memoryPrint)
/ availableMemory);
return divisions;
}
void
PipelineMemoryPrintCalculator
::PrintSelf(std::ostream& os, itk::Indent indent) const
......@@ -52,8 +62,6 @@ PipelineMemoryPrintCalculator
// Display parameters
os<<indent<<"Data to write: "<<m_DataToWrite<<std::endl;
os<<indent<<"Memory print of whole pipeline: "<<m_MemoryPrint * ByteToMegabyte <<" Mb"<<std::endl;
os<<indent<<"Available memory: "<<m_AvailableMemory * ByteToMegabyte <<" Mb"<<std::endl;
os<<indent<<"Optimal number of stream divisions: "<<m_OptimalNumberOfStreamDivisions<<std::endl;
os<<indent<<"Bias correction factor applied: "<<m_BiasCorrectionFactor<<std::endl;
}
......@@ -76,7 +84,7 @@ PipelineMemoryPrintCalculator
if(source)
{
// Call the recursive memory print evaluation
m_MemoryPrint = EvaluateMemoryPrint(source);
m_MemoryPrint = EvaluateProcessObjectPrintRecursive(source);
}
else
{
......@@ -87,14 +95,11 @@ PipelineMemoryPrintCalculator
// Apply bias correction factor
m_MemoryPrint *= m_BiasCorrectionFactor;
// Compute the optimal number of stream division
m_OptimalNumberOfStreamDivisions = vcl_ceil(static_cast<double>(m_MemoryPrint)
/m_AvailableMemory);
}
PipelineMemoryPrintCalculator::MemoryPrintType
PipelineMemoryPrintCalculator
::EvaluateMemoryPrint(ProcessObjectType * process)
::EvaluateProcessObjectPrintRecursive(ProcessObjectType * process)
{
otbMsgDevMacro(<< "EvaluateMemoryPrint for " << process->GetNameOfClass() << " (" << process << ")")
// This variable will store the final print
......@@ -127,7 +132,7 @@ PipelineMemoryPrintCalculator
// If data object has a source
if(source)
{
print += this->EvaluateMemoryPrint(source);
print += this->EvaluateProcessObjectPrintRecursive(source);
}
else
{
......@@ -136,6 +141,7 @@ PipelineMemoryPrintCalculator
}
}
}
// Retrieve the output array
ProcessObjectType::DataObjectPointerArray outputs = process->GetOutputs();
......
......@@ -32,24 +32,24 @@ namespace otb
* data should be set to the data to write) and
* examining each filter to determine its memory footprint. To do so,
* it performs a dry run of the requested region pipeline
* negociation.
* negotiation.
*
* The SetDataToWrite() method allows to set the data candidate for
* writing, and for which memory usage estimation should be
* performed.
*
* Additionnaly, this class allows to compute the optimal number of
* Additionally, this class allows to compute the optimal number of
* stream division to write the data. To do so, the available memory
* can be set via the SetAvailableMemory() method, and an optionnal
* bias correction factor can be applied to wheight the estimate
* can be set via the SetAvailableMemory() method, and an optional
* bias correction factor can be applied to weight the estimate
* memory usage in case a bias occurs between estimated and real
* memory usage. The optimal number of stream divisions can be
* retreived using the GetOptimalNumberOfStreamDivisions().
* retrieved using the GetOptimalNumberOfStreamDivisions().
*
* Please note that for now this calculator suffers from the
* following limitations:
* - DataObject taken into account for memory usage estimation are
* only Image and VectorImage instantiation,
* only Image, VectorImage and ImageList instantiation,
* - The estimator is non-intrusive regarding the pipeline high-level
* class, but it is not able to estimate memory usage of minipipeline
* within composite filter (because there is no way to trace back to
......@@ -85,11 +85,6 @@ public:
/** Get the total memory print (in bytes) */
itkGetMacro(MemoryPrint, MemoryPrintType);
/** Set/Get the available memory for pipeline execution (in bytes,
* default is 256 Mb) */
itkSetMacro(AvailableMemory, MemoryPrintType);
itkGetMacro(AvailableMemory, MemoryPrintType);
/** Set/Get the bias correction factor which will weight the
* estimated memory print (allows to compensate bias between
* estimated and real memory print, default is 1., i.e. no correction) */
......@@ -97,7 +92,8 @@ public:
itkGetMacro(BiasCorrectionFactor, double);
/** Get the optimal number of stream division */
itkGetMacro(OptimalNumberOfStreamDivisions, unsigned long);
static unsigned long EstimateOptimalNumberOfStreamDivisions(
MemoryPrintType memoryPrint, MemoryPrintType availableMemory);
/** Set last pipeline filter */
itkSetObjectMacro(DataToWrite, DataObjectType);
......@@ -107,6 +103,10 @@ public:
/** Const conversion factor */
static const double ByteToMegabyte;
static const double MegabyteToByte;
/** Evaluate the print (in bytes) of a single data object */
MemoryPrintType EvaluateDataObjectPrint(DataObjectType * data) const;
protected:
/** Constructor */
......@@ -118,11 +118,8 @@ protected:
/** PrintSelf method */
void PrintSelf(std::ostream& os, itk::Indent indent) const;
/** Internal recursive method to evaluate memory print in bytes */
MemoryPrintType EvaluateMemoryPrint(ProcessObjectType * process);
/** Internal method to evaluate the print (in Mo) of a single data object */
MemoryPrintType EvaluateDataObjectPrint(DataObjectType * data) const;
/** Recursive method to evaluate memory print in bytes */
MemoryPrintType EvaluateProcessObjectPrintRecursive(ProcessObjectType * process);
private:
PipelineMemoryPrintCalculator(const Self &); //purposely not implemented
......@@ -131,12 +128,6 @@ private:
/** The total memory print of the pipeline */
MemoryPrintType m_MemoryPrint;
/** The available memory for pipeline execution */
MemoryPrintType m_AvailableMemory;
/** The optimal number of stream division */
unsigned long m_OptimalNumberOfStreamDivisions;
/** Pointer to the last pipeline filter */
DataObjectPointerType m_DataToWrite;
......
......@@ -24,6 +24,7 @@
#include "itkObject.h"
#include "itkDataObject.h"
#include "itkImageRegionSplitter.h"
#include "otbPipelineMemoryPrintCalculator.h"
namespace otb
{
......@@ -62,6 +63,8 @@ public:
typedef typename RegionType::SizeType SizeType;
typedef typename ImageType::InternalPixelType PixelType;
typedef otb::PipelineMemoryPrintCalculator::MemoryPrintType MemoryPrintType;
/** Type macro */
itkTypeMacro(StreamingManager, itk::LightObject);
......@@ -87,7 +90,7 @@ protected:
virtual ~StreamingManager();
virtual unsigned int EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
unsigned int availableRAM);
MemoryPrintType availableRAMInMB);
/** The number of splits generated by the splitter */
unsigned int m_ComputedNumberOfSplits;
......@@ -99,10 +102,16 @@ protected:
typedef itk::ImageRegionSplitter<itkGetStaticConstMacro(ImageDimension)> AbstractSplitterType;
typedef typename AbstractSplitterType::Pointer AbstractSplitterPointerType;
AbstractSplitterPointerType m_Splitter;
private:
StreamingManager(const StreamingManager &); //purposely not implemented
void operator =(const StreamingManager&); //purposely not implemented
/* Compute the available RAM from configuration settings if the input parameter is 0,
* otherwise, simply returns the input parameter */
MemoryPrintType GetActualAvailableRAMInBytes(MemoryPrintType availableRAMInMB);
};
} // End namespace otb
......
......@@ -22,7 +22,6 @@
#include "otbMacro.h"
#include "otbConfigure.h"
#include "otbConfigurationFile.h"
#include "otbPipelineMemoryPrintCalculator.h"
#include "itkExtractImageFilter.h"
namespace otb
......@@ -40,12 +39,10 @@ StreamingManager<TImage>::~StreamingManager()
}
template <class TImage>
unsigned int
StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
unsigned int availableRAM)
typename StreamingManager<TImage>::MemoryPrintType
StreamingManager<TImage>::GetActualAvailableRAMInBytes(MemoryPrintType availableRAMInMB)
{
otbMsgDevMacro(<< "availableRAM " << availableRAM)
unsigned int availableRAMInBytes = availableRAM * 1024 * 1024;
unsigned int availableRAMInBytes = availableRAMInMB * 1024 * 1024;
if (availableRAMInBytes == 0)
{
......@@ -63,24 +60,34 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
{
// We should never have to go here if the configuration file is
// correct and found.
// In case it is not fallback on the cmake
// In case it is not, fallback on the cmake
// defined constants.
availableRAMInBytes = OTB_STREAM_MAX_SIZE_BUFFER_FOR_STREAMING;
}
}
otbMsgDevMacro("RAM used to estimate memory footprint : " << availableRAMInBytes / 1024 / 1024 << " MB")
return availableRAMInBytes;
}
template <class TImage>
unsigned int
StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
MemoryPrintType availableRAM)
{
otbMsgDevMacro(<< "availableRAM " << availableRAM)
MemoryPrintType availableRAMInBytes = GetActualAvailableRAMInBytes(availableRAM);
otb::PipelineMemoryPrintCalculator::Pointer memoryPrintCalculator;
memoryPrintCalculator = otb::PipelineMemoryPrintCalculator::New();
memoryPrintCalculator->SetAvailableMemory( availableRAMInBytes );
// Trick to avoid having the resampler compute the whole
// deformation field
double regionTrickFactor = 1;
ImageType* inputImage = dynamic_cast<ImageType*>(input);
//inputImage = 0;
MemoryPrintType pipelineMemoryPrint;
if (inputImage)
{
......@@ -127,6 +134,17 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
}
memoryPrintCalculator->Compute();
pipelineMemoryPrint = memoryPrintCalculator->GetMemoryPrint();
if (smallRegionSuccess)
{
// remove the contribution of the ExtractImageFilter
MemoryPrintType extractContrib =
memoryPrintCalculator->EvaluateDataObjectPrint(extractFilter->GetOutput());
pipelineMemoryPrint -= extractContrib;
}
}
else
{
......@@ -135,14 +153,19 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
memoryPrintCalculator->SetBiasCorrectionFactor(1.0);
memoryPrintCalculator->Compute();
pipelineMemoryPrint = memoryPrintCalculator->GetMemoryPrint();
}
unsigned int optimalNumberOfDivisions =
otb::PipelineMemoryPrintCalculator::EstimateOptimalNumberOfStreamDivisions(pipelineMemoryPrint, availableRAMInBytes);
otbMsgDevMacro( "Estimated Memory print for the full image : "
<< static_cast<unsigned int>(memoryPrintCalculator->GetMemoryPrint() / 1024 / 1024 ) << std::endl)
<< static_cast<unsigned int>(pipelineMemoryPrint * otb::PipelineMemoryPrintCalculator::ByteToMegabyte ) << std::endl)
otbMsgDevMacro( "Optimal number of stream divisions: "
<< memoryPrintCalculator->GetOptimalNumberOfStreamDivisions() << std::endl)
<< optimalNumberOfDivisions << std::endl)
return memoryPrintCalculator->GetOptimalNumberOfStreamDivisions();
return optimalNumberOfDivisions;
}
template <class TImage>
......
......@@ -34,7 +34,7 @@ int otbPipelineMemoryPrintCalculatorTest(int argc, char * argv[])
{
typedef otb::VectorImage<double, 2> VectorImageType;
typedef otb::Image<double, 2> ImageType;
typedef otb::ImageFileReader<VectorImageType> ReaderType;
typedef otb::ImageFileReader<VectorImageType> ReaderType;
typedef otb::VectorImageToIntensityImageFilter
<VectorImageType, ImageType> IntensityImageFilterType;
......@@ -46,15 +46,19 @@ int otbPipelineMemoryPrintCalculatorTest(int argc, char * argv[])
otb::PipelineMemoryPrintCalculator::Pointer calculator = otb::PipelineMemoryPrintCalculator::New();
calculator->SetDataToWrite(intensity->GetOutput());
calculator->SetAvailableMemory(104858);
calculator->Compute();
otb::PipelineMemoryPrintCalculator::MemoryPrintType availableMemoryInBytes = 104858;
otb::PipelineMemoryPrintCalculator::MemoryPrintType memoryPrint = calculator->GetMemoryPrint();
unsigned int optimalNbDiv = otb::PipelineMemoryPrintCalculator::EstimateOptimalNumberOfStreamDivisions(memoryPrint, availableMemoryInBytes);
std::ofstream ofs(argv[2]);
ofs<<"Memory print of whole pipeline: "<<calculator->GetMemoryPrint()
* otb::PipelineMemoryPrintCalculator::ByteToMegabyte << " Mb"<<std::endl;
ofs<<"Available memory: "<<calculator->GetAvailableMemory()
ofs<<"Available memory: "<<availableMemoryInBytes
* otb::PipelineMemoryPrintCalculator::ByteToMegabyte << " Mb"<<std::endl;
ofs<<"Optimal number of stream divisions: "<<calculator->GetOptimalNumberOfStreamDivisions()<<std::endl;
ofs<<"Optimal number of stream divisions: "<<optimalNbDiv<<std::endl;
ofs<<"Bias correction factor applied: "<<calculator->GetBiasCorrectionFactor()<<std::endl;
ofs.close();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment