ENH: remove contribution of ExtractROI in memory print estimation

Code/Common/otbPipelineMemoryPrintCalculator.cxx

@@ -27,12 +27,11 @@
 namespace otb
 {
 const double PipelineMemoryPrintCalculator::ByteToMegabyte = 1./vcl_pow(2.0, 20);
+const double PipelineMemoryPrintCalculator::MegabyteToByte = vcl_pow(2.0, 20);
 
 PipelineMemoryPrintCalculator
 ::PipelineMemoryPrintCalculator()
   : m_MemoryPrint(0),
-    m_AvailableMemory(256000000),
-    m_OptimalNumberOfStreamDivisions(1),
     m_DataToWrite(NULL),
     m_BiasCorrectionFactor(1.),
     m_VisitedProcessObjects()
@@ -42,6 +41,17 @@ PipelineMemoryPrintCalculator
 ::~PipelineMemoryPrintCalculator()
 {}
 
+// [static]
+unsigned long
+PipelineMemoryPrintCalculator
+::EstimateOptimalNumberOfStreamDivisions(MemoryPrintType memoryPrint, MemoryPrintType availableMemory)
+{
+  unsigned long divisions;
+  divisions = vcl_ceil(static_cast<double>(memoryPrint)
+                       / availableMemory);
+  return divisions;
+}
+
 void
 PipelineMemoryPrintCalculator
 ::PrintSelf(std::ostream& os, itk::Indent indent) const
@@ -52,8 +62,6 @@ PipelineMemoryPrintCalculator
   // Display parameters
   os<<indent<<"Data to write:                      "<<m_DataToWrite<<std::endl;
   os<<indent<<"Memory print of whole pipeline:     "<<m_MemoryPrint * ByteToMegabyte <<" Mb"<<std::endl;
-  os<<indent<<"Available memory:                   "<<m_AvailableMemory * ByteToMegabyte <<" Mb"<<std::endl;
-  os<<indent<<"Optimal number of stream divisions: "<<m_OptimalNumberOfStreamDivisions<<std::endl;
   os<<indent<<"Bias correction factor applied:     "<<m_BiasCorrectionFactor<<std::endl;
 }
 
@@ -76,7 +84,7 @@ PipelineMemoryPrintCalculator
   if(source)
     {
     // Call the recursive memory print evaluation
-    m_MemoryPrint = EvaluateMemoryPrint(source);
+    m_MemoryPrint = EvaluateProcessObjectPrintRecursive(source);
     }
   else
     {
@@ -87,14 +95,11 @@ PipelineMemoryPrintCalculator
   // Apply bias correction factor
   m_MemoryPrint *= m_BiasCorrectionFactor;
 
-  // Compute the optimal number of stream division
-  m_OptimalNumberOfStreamDivisions = vcl_ceil(static_cast<double>(m_MemoryPrint)
-                                              /m_AvailableMemory);
 }
 
 PipelineMemoryPrintCalculator::MemoryPrintType
 PipelineMemoryPrintCalculator
-::EvaluateMemoryPrint(ProcessObjectType * process)
+::EvaluateProcessObjectPrintRecursive(ProcessObjectType * process)
 {
   otbMsgDevMacro(<< "EvaluateMemoryPrint for " << process->GetNameOfClass() << " (" << process << ")")
   // This variable will store the final print
@@ -127,7 +132,7 @@ PipelineMemoryPrintCalculator
         // If data object has a source
         if(source)
           {
-            print += this->EvaluateMemoryPrint(source);
+            print += this->EvaluateProcessObjectPrintRecursive(source);
           }
         else
           {
@@ -136,6 +141,7 @@ PipelineMemoryPrintCalculator
           }
       }
     }
+
   // Retrieve the output array
   ProcessObjectType::DataObjectPointerArray outputs = process->GetOutputs();
 

Code/Common/otbPipelineMemoryPrintCalculator.h

@@ -32,24 +32,24 @@ namespace otb
  *  data should be set to the data to write) and
  *  examining each filter to determine its memory footprint. To do so,
  *  it performs a dry run of the requested region pipeline
- *  negociation.
+ *  negotiation.
  *
  *  The SetDataToWrite() method allows to set the data candidate for
  *  writing, and for which memory usage estimation should be
  *  performed.
  *
- *  Additionnaly, this class allows to compute the optimal number of
+ *  Additionally, this class allows to compute the optimal number of
  *  stream division to write the data. To do so, the available memory
- *  can be set via the SetAvailableMemory() method, and an optionnal
- *  bias correction factor can be applied to wheight the estimate
+ *  can be set via the SetAvailableMemory() method, and an optional
+ *  bias correction factor can be applied to weight the estimate
  *  memory usage in case a bias occurs between estimated and real
  *  memory usage. The optimal number of stream divisions can be
- *  retreived using the GetOptimalNumberOfStreamDivisions().
+ *  retrieved using the GetOptimalNumberOfStreamDivisions().
  *
  *  Please note that for now this calculator suffers from the
  *  following limitations:
  *  - DataObject taken into account for memory usage estimation are
- *  only Image and VectorImage instantiation,
+ *  only Image, VectorImage and ImageList instantiation,
  *  - The estimator is non-intrusive regarding the pipeline high-level
  *  class, but it is not able to estimate memory usage of minipipeline
  *  within composite filter (because there is no way to trace back to
@@ -85,11 +85,6 @@ public:
   /** Get the total memory print (in bytes) */
   itkGetMacro(MemoryPrint, MemoryPrintType);
 
-  /** Set/Get the available memory for pipeline execution (in bytes,
-   *  default is 256 Mb) */
-  itkSetMacro(AvailableMemory, MemoryPrintType);
-  itkGetMacro(AvailableMemory, MemoryPrintType);
-
   /** Set/Get the bias correction factor which will weight the
    * estimated memory print (allows to compensate bias between
    * estimated and real memory print, default is 1., i.e. no correction) */
@@ -97,7 +92,8 @@ public:
   itkGetMacro(BiasCorrectionFactor, double);
 
   /** Get the optimal number of stream division */
-  itkGetMacro(OptimalNumberOfStreamDivisions, unsigned long);
+  static unsigned long EstimateOptimalNumberOfStreamDivisions(
+      MemoryPrintType memoryPrint, MemoryPrintType availableMemory);
 
   /** Set last pipeline filter */
   itkSetObjectMacro(DataToWrite, DataObjectType);
@@ -107,6 +103,10 @@ public:
 
   /** Const conversion factor */
   static const double ByteToMegabyte;
+  static const double MegabyteToByte;
+
+  /** Evaluate the print (in bytes) of a single data object */
+  MemoryPrintType EvaluateDataObjectPrint(DataObjectType * data) const;
 
 protected:
   /** Constructor */
@@ -118,11 +118,8 @@ protected:
   /** PrintSelf method */
   void PrintSelf(std::ostream& os, itk::Indent indent) const;
 
-  /** Internal recursive method to evaluate memory print in bytes */
-  MemoryPrintType EvaluateMemoryPrint(ProcessObjectType * process);
-
-  /** Internal method to evaluate the print (in Mo) of a single data object */
-  MemoryPrintType EvaluateDataObjectPrint(DataObjectType * data) const;
+  /** Recursive method to evaluate memory print in bytes */
+  MemoryPrintType EvaluateProcessObjectPrintRecursive(ProcessObjectType * process);
 
 private:
   PipelineMemoryPrintCalculator(const Self &); //purposely not implemented
@@ -131,12 +128,6 @@ private:
   /** The total memory print of the pipeline */
   MemoryPrintType       m_MemoryPrint;
 
-  /** The available memory for pipeline execution */
-  MemoryPrintType       m_AvailableMemory;
-
-  /** The optimal number of stream division */
-  unsigned long m_OptimalNumberOfStreamDivisions;
-
   /** Pointer to the last pipeline filter */
   DataObjectPointerType m_DataToWrite;
 

Code/Common/otbStreamingManager.h

@@ -24,6 +24,7 @@
 #include "itkObject.h"
 #include "itkDataObject.h"
 #include "itkImageRegionSplitter.h"
+#include "otbPipelineMemoryPrintCalculator.h"
 
 namespace otb
 {
@@ -62,6 +63,8 @@ public:
   typedef typename RegionType::SizeType         SizeType;
   typedef typename ImageType::InternalPixelType PixelType;
 
+  typedef otb::PipelineMemoryPrintCalculator::MemoryPrintType MemoryPrintType;
+
   /** Type macro */
   itkTypeMacro(StreamingManager, itk::LightObject);
 
@@ -87,7 +90,7 @@ protected:
   virtual ~StreamingManager();
 
   virtual unsigned int EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
-                                                        unsigned int availableRAM);
+                                                        MemoryPrintType availableRAMInMB);
 
   /** The number of splits generated by the splitter */
   unsigned int m_ComputedNumberOfSplits;
@@ -99,10 +102,16 @@ protected:
   typedef itk::ImageRegionSplitter<itkGetStaticConstMacro(ImageDimension)> AbstractSplitterType;
   typedef typename AbstractSplitterType::Pointer                           AbstractSplitterPointerType;
   AbstractSplitterPointerType m_Splitter;
+
 private:
   StreamingManager(const StreamingManager &); //purposely not implemented
   void operator =(const StreamingManager&);   //purposely not implemented
 
+  /* Compute the available RAM from configuration settings if the input parameter is 0,
+   * otherwise, simply returns the input parameter */
+  MemoryPrintType GetActualAvailableRAMInBytes(MemoryPrintType availableRAMInMB);
+
+
 };
 
 } // End namespace otb

Code/Common/otbStreamingManager.txx

@@ -22,7 +22,6 @@
 #include "otbMacro.h"
 #include "otbConfigure.h"
 #include "otbConfigurationFile.h"
-#include "otbPipelineMemoryPrintCalculator.h"
 #include "itkExtractImageFilter.h"
 
 namespace otb
@@ -40,12 +39,10 @@ StreamingManager<TImage>::~StreamingManager()
 }
 
 template <class TImage>
-unsigned int
-StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
-                                                           unsigned int availableRAM)
+typename StreamingManager<TImage>::MemoryPrintType
+StreamingManager<TImage>::GetActualAvailableRAMInBytes(MemoryPrintType availableRAMInMB)
 {
-  otbMsgDevMacro(<< "availableRAM " << availableRAM)
-  unsigned int availableRAMInBytes = availableRAM * 1024 * 1024;
+  unsigned int availableRAMInBytes = availableRAMInMB * 1024 * 1024;
 
   if (availableRAMInBytes == 0)
     {
@@ -63,24 +60,34 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
       {
       // We should never have to go here if the configuration file is
       // correct and found.
-      // In case it is not fallback on the cmake
+      // In case it is not, fallback on the cmake
       // defined constants.
       availableRAMInBytes = OTB_STREAM_MAX_SIZE_BUFFER_FOR_STREAMING;
       }
     }
 
   otbMsgDevMacro("RAM used to estimate memory footprint : " << availableRAMInBytes / 1024 / 1024  << " MB")
+  return availableRAMInBytes;
+}
+
+template <class TImage>
+unsigned int
+StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
+                                                           MemoryPrintType availableRAM)
+{
+  otbMsgDevMacro(<< "availableRAM " << availableRAM)
+
+  MemoryPrintType availableRAMInBytes = GetActualAvailableRAMInBytes(availableRAM);
 
   otb::PipelineMemoryPrintCalculator::Pointer memoryPrintCalculator;
   memoryPrintCalculator = otb::PipelineMemoryPrintCalculator::New();
 
-  memoryPrintCalculator->SetAvailableMemory( availableRAMInBytes );
-
   // Trick to avoid having the resampler compute the whole
   // deformation field
   double     regionTrickFactor = 1;
   ImageType* inputImage = dynamic_cast<ImageType*>(input);
-  //inputImage = 0;
+
+  MemoryPrintType pipelineMemoryPrint;
   if (inputImage)
     {
 
@@ -127,6 +134,17 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
       }
 
     memoryPrintCalculator->Compute();
+
+    pipelineMemoryPrint = memoryPrintCalculator->GetMemoryPrint();
+
+    if (smallRegionSuccess)
+      {
+      // remove the contribution of the ExtractImageFilter
+      MemoryPrintType extractContrib =
+          memoryPrintCalculator->EvaluateDataObjectPrint(extractFilter->GetOutput());
+
+      pipelineMemoryPrint -= extractContrib;
+      }
     }
   else
     {
@@ -135,14 +153,19 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
     memoryPrintCalculator->SetBiasCorrectionFactor(1.0);
 
     memoryPrintCalculator->Compute();
+
+    pipelineMemoryPrint = memoryPrintCalculator->GetMemoryPrint();
     }
 
+  unsigned int optimalNumberOfDivisions =
+      otb::PipelineMemoryPrintCalculator::EstimateOptimalNumberOfStreamDivisions(pipelineMemoryPrint, availableRAMInBytes);
+
   otbMsgDevMacro( "Estimated Memory print for the full image : "
-                  << static_cast<unsigned int>(memoryPrintCalculator->GetMemoryPrint() / 1024 / 1024 ) << std::endl)
+                  << static_cast<unsigned int>(pipelineMemoryPrint * otb::PipelineMemoryPrintCalculator::ByteToMegabyte ) << std::endl)
   otbMsgDevMacro( "Optimal number of stream divisions: "
-                  << memoryPrintCalculator->GetOptimalNumberOfStreamDivisions() << std::endl)
+                  << optimalNumberOfDivisions << std::endl)
 
-  return memoryPrintCalculator->GetOptimalNumberOfStreamDivisions();
+  return optimalNumberOfDivisions;
 }
 
 template <class TImage>

Testing/Code/Common/otbPipelineMemoryPrintCalculatorTest.cxx

@@ -34,7 +34,7 @@ int otbPipelineMemoryPrintCalculatorTest(int argc, char * argv[])
 {
   typedef otb::VectorImage<double, 2>            VectorImageType;
   typedef otb::Image<double, 2>                  ImageType;
-  typedef otb::ImageFileReader<VectorImageType> ReaderType;
+  typedef otb::ImageFileReader<VectorImageType>  ReaderType;
   typedef otb::VectorImageToIntensityImageFilter
     <VectorImageType, ImageType>                 IntensityImageFilterType;
 
@@ -46,15 +46,19 @@ int otbPipelineMemoryPrintCalculatorTest(int argc, char * argv[])
 
   otb::PipelineMemoryPrintCalculator::Pointer calculator = otb::PipelineMemoryPrintCalculator::New();
   calculator->SetDataToWrite(intensity->GetOutput());
-  calculator->SetAvailableMemory(104858);
   calculator->Compute();
 
+  otb::PipelineMemoryPrintCalculator::MemoryPrintType availableMemoryInBytes = 104858;
+  otb::PipelineMemoryPrintCalculator::MemoryPrintType memoryPrint = calculator->GetMemoryPrint();
+
+  unsigned int optimalNbDiv = otb::PipelineMemoryPrintCalculator::EstimateOptimalNumberOfStreamDivisions(memoryPrint, availableMemoryInBytes);
+
   std::ofstream ofs(argv[2]);
   ofs<<"Memory print of whole pipeline:     "<<calculator->GetMemoryPrint()
     * otb::PipelineMemoryPrintCalculator::ByteToMegabyte << " Mb"<<std::endl;
-  ofs<<"Available memory:                   "<<calculator->GetAvailableMemory()
+  ofs<<"Available memory:                   "<<availableMemoryInBytes
     * otb::PipelineMemoryPrintCalculator::ByteToMegabyte << " Mb"<<std::endl;
-  ofs<<"Optimal number of stream divisions: "<<calculator->GetOptimalNumberOfStreamDivisions()<<std::endl;
+  ofs<<"Optimal number of stream divisions: "<<optimalNbDiv<<std::endl;
   ofs<<"Bias correction factor applied:     "<<calculator->GetBiasCorrectionFactor()<<std::endl;
   ofs.close();