ENH: make the bias parameter of PipelineMemoryPrintEstimator available through the StreamingManager

Code/Common/otbRAMDrivenStrippedStreamingManager.h

@@ -62,7 +62,13 @@ public:
 
   /** The number of Megabytes available (if 0, the configuration option is
     used)*/
-  itkGetMacro(AvailableRAMInMB, unsigned int);
+  itkGetConstMacro(AvailableRAMInMB, unsigned int);
+
+  /** The multiplier to apply to the memory print estimation */
+  itkSetMacro(Bias, double);
+
+  /** The multiplier to apply to the memory print estimation */
+  itkGetConstMacro(Bias, double);
 
   /** Actually computes the stream divisions, according to the specified streaming mode,
    * eventually using the input parameter to estimate memory consumption */
@@ -77,6 +83,10 @@ protected:
 
   /** The number of MegaBytes of RAM available */
   unsigned int m_AvailableRAMInMB;
+
+  /** The multiplier to apply to the memory print estimation */
+  double m_Bias;
+
 private:
   RAMDrivenStrippedStreamingManager(const RAMDrivenStrippedStreamingManager &);
   void operator =(const RAMDrivenStrippedStreamingManager&);

Code/Common/otbRAMDrivenStrippedStreamingManager.txx

@@ -26,7 +26,8 @@ namespace otb
 
 template <class TImage>
 RAMDrivenStrippedStreamingManager<TImage>::RAMDrivenStrippedStreamingManager()
-  : m_AvailableRAMInMB(0)
+  : m_AvailableRAMInMB(0),
+    m_Bias(1.0)
 {
 }
 
@@ -39,7 +40,8 @@ template <class TImage>
 void
 RAMDrivenStrippedStreamingManager<TImage>::PrepareStreaming( itk::DataObject * input, const RegionType &region )
 {
-  unsigned long nbDivisions = this->EstimateOptimalNumberOfDivisions(input, region, m_AvailableRAMInMB);
+  unsigned long nbDivisions =
+      this->EstimateOptimalNumberOfDivisions(input, region, m_AvailableRAMInMB, m_Bias);
 
   this->m_Splitter = itk::ImageRegionSplitter<itkGetStaticConstMacro(ImageDimension)>::New();
   this->m_ComputedNumberOfSplits = this->m_Splitter->GetNumberOfSplits(region, nbDivisions);

Code/Common/otbRAMDrivenTiledStreamingManager.h

@@ -62,7 +62,13 @@ public:
 
   /** The number of Megabytes available (if 0, the configuration option is
     used)*/
-  itkGetMacro(AvailableRAMInMB, unsigned int);
+  itkGetConstMacro(AvailableRAMInMB, unsigned int);
+
+  /** The multiplier to apply to the memory print estimation */
+  itkSetMacro(Bias, double);
+
+  /** The multiplier to apply to the memory print estimation */
+  itkGetConstMacro(Bias, double);
 
   /** Actually computes the stream divisions, according to the specified streaming mode,
    * eventually using the input parameter to estimate memory consumption */
@@ -74,6 +80,10 @@ protected:
 
   /** The number of MegaBytes of RAM available */
   unsigned int m_AvailableRAMInMB;
+
+  /** The multiplier to apply to the memory print estimation */
+  double m_Bias;
+
 private:
   RAMDrivenTiledStreamingManager(const RAMDrivenTiledStreamingManager &);
   void operator =(const RAMDrivenTiledStreamingManager&);

Code/Common/otbRAMDrivenTiledStreamingManager.txx

@@ -27,7 +27,8 @@ namespace otb
 
 template <class TImage>
 RAMDrivenTiledStreamingManager<TImage>::RAMDrivenTiledStreamingManager()
-  : m_AvailableRAMInMB(0)
+  : m_AvailableRAMInMB(0),
+    m_Bias(1.0)
 {
 }
 
@@ -40,7 +41,8 @@ template <class TImage>
 void
 RAMDrivenTiledStreamingManager<TImage>::PrepareStreaming( itk::DataObject * input, const RegionType &region )
 {
-  unsigned long nbDivisions = this->EstimateOptimalNumberOfDivisions(input, region, m_AvailableRAMInMB);
+  unsigned long nbDivisions =
+      this->EstimateOptimalNumberOfDivisions(input, region, m_AvailableRAMInMB, m_Bias);
 
   this->m_Splitter = otb::ImageRegionSquareTileSplitter<itkGetStaticConstMacro(ImageDimension)>::New();
   this->m_ComputedNumberOfSplits = this->m_Splitter->GetNumberOfSplits(region, nbDivisions);

Code/Common/otbStreamingManager.h

@@ -90,7 +90,8 @@ protected:
   virtual ~StreamingManager();
 
   virtual unsigned int EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
-                                                        MemoryPrintType availableRAMInMB);
+                                                        MemoryPrintType availableRAMInMB,
+                                                        double bias = 1.0);
 
   /** The number of splits generated by the splitter */
   unsigned int m_ComputedNumberOfSplits;

Code/Common/otbStreamingManager.txx

@@ -73,7 +73,8 @@ StreamingManager<TImage>::GetActualAvailableRAMInBytes(MemoryPrintType available
 template <class TImage>
 unsigned int
 StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * input, const RegionType &region,
-                                                           MemoryPrintType availableRAM)
+                                                           MemoryPrintType availableRAM,
+                                                           double bias)
 {
   otbMsgDevMacro(<< "availableRAM " << availableRAM)
 
@@ -119,10 +120,11 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
       otbMsgDevMacro("Using an extract to estimate memory : " << smallRegion)
       // the region is well behaved, inside the largest possible region
       memoryPrintCalculator->SetDataToWrite(extractFilter->GetOutput() );
+
       regionTrickFactor = static_cast<double>( region.GetNumberOfPixels() )
         / static_cast<double>(smallRegion.GetNumberOfPixels() );
 
-      memoryPrintCalculator->SetBiasCorrectionFactor(regionTrickFactor);
+      memoryPrintCalculator->SetBiasCorrectionFactor(regionTrickFactor * bias);
       }
     else
       {
@@ -130,7 +132,7 @@ StreamingManager<TImage>::EstimateOptimalNumberOfDivisions(itk::DataObject * inp
       // the region is not well behaved
       // use the full region
       memoryPrintCalculator->SetDataToWrite(input);
-      memoryPrintCalculator->SetBiasCorrectionFactor(1.0);
+      memoryPrintCalculator->SetBiasCorrectionFactor(bias);
       }
 
     memoryPrintCalculator->Compute();

Code/IO/otbStreamingImageFileWriter.h

@@ -113,8 +113,12 @@ public:
    *   available. The actual number of divisions is computed automatically
    *   by estimating the memory consumption of the pipeline.
    *   Setting the availableRAM parameter to 0 means that the available RAM
-   *   is set from the CMake configuration option */
-  void SetAutomaticStrippedStreaming(unsigned int availableRAM = 0);
+   *   is set from the CMake configuration option.
+   *   The bias parameter is a multiplier applied on the estimated memory size
+   *   of the pipeline and can be used to fine tune the potential gap between
+   *   estimated memory and actual memory used, which can happen because of
+   *   composite filters for example */
+  void SetAutomaticStrippedStreaming(unsigned int availableRAM = 0, double bias = 1.0);
 
   /**  Set the streaming mode to 'tiled' and configure the dimension of the tiles
    *   in pixels for each dimension (square tiles will be generated) */
@@ -125,8 +129,12 @@ public:
    *   by estimating the memory consumption of the pipeline.
    *   Tiles will be square.
    *   Setting the availableRAM parameter to 0 means that the available RAM
-   *   is set from the CMake configuration option */
-  void SetAutomaticTiledStreaming(unsigned int availableRAM = 0);
+   *   is set from the CMake configuration option
+   *   The bias parameter is a multiplier applied on the estimated memory size
+   *   of the pipeline and can be used to fine tune the potential gap between
+   *   estimated memory and actual memory used, which can happen because of
+   *   composite filters for example */
+  void SetAutomaticTiledStreaming(unsigned int availableRAM = 0, double bias = 1.0);
 
   /**  Set buffer memory size (in bytes) use to calculate the number of stream divisions */
   itkLegacyMacro( void SetBufferMemorySize(unsigned long) );

Code/IO/otbStreamingImageFileWriter.txx

@@ -107,11 +107,12 @@ StreamingImageFileWriter<TInputImage>
 template <class TInputImage>
 void
 StreamingImageFileWriter<TInputImage>
-::SetAutomaticStrippedStreaming(unsigned int availableRAM)
+::SetAutomaticStrippedStreaming(unsigned int availableRAM, double bias)
 {
   typedef RAMDrivenStrippedStreamingManager<TInputImage> RAMDrivenStrippedStreamingManagerType;
   typename RAMDrivenStrippedStreamingManagerType::Pointer streamingManager = RAMDrivenStrippedStreamingManagerType::New();
   streamingManager->SetAvailableRAMInMB(availableRAM);
+  streamingManager->SetBias(bias);
 
   m_StreamingManager = streamingManager;
 }
@@ -131,12 +132,12 @@ StreamingImageFileWriter<TInputImage>
 template <class TInputImage>
 void
 StreamingImageFileWriter<TInputImage>
-::SetAutomaticTiledStreaming(unsigned int availableRAM)
+::SetAutomaticTiledStreaming(unsigned int availableRAM, double bias)
 {
   typedef RAMDrivenTiledStreamingManager<TInputImage> RAMDrivenTiledStreamingManagerType;
   typename RAMDrivenTiledStreamingManagerType::Pointer streamingManager = RAMDrivenTiledStreamingManagerType::New();
   streamingManager->SetAvailableRAMInMB(availableRAM);
-
+  streamingManager->SetBias(bias);
   m_StreamingManager = streamingManager;
 }
 

Code/IO/otbStreamingImageVirtualWriter.h

@@ -111,7 +111,7 @@ public:
    *   by estimating the memory consumption of the pipeline.
    *   Setting the availableRAM parameter to 0 means that the available RAM
    *   is set from the CMake configuration option */
-  void SetAutomaticStrippedStreaming(unsigned int availableRAM);
+  void SetAutomaticStrippedStreaming(unsigned int availableRAM, double bias = 1.0);
 
   /**  Set the streaming mode to 'tiled' and configure the dimension of the tiles
    *   in pixels for each dimension (square tiles will be generated) */
@@ -123,7 +123,7 @@ public:
    *   Tiles will be square.
    *   Setting the availableRAM parameter to 0 means that the available RAM
    *   is set from the CMake configuration option */
-  void SetAutomaticTiledStreaming(unsigned int availableRAM);
+  void SetAutomaticTiledStreaming(unsigned int availableRAM, double bias = 1.0);
 
 protected:
   StreamingImageVirtualWriter();

Code/IO/otbStreamingImageVirtualWriter.txx

@@ -95,12 +95,12 @@ StreamingImageVirtualWriter<TInputImage>
 template <class TInputImage>
 void
 StreamingImageVirtualWriter<TInputImage>
-::SetAutomaticStrippedStreaming(unsigned int availableRAM)
+::SetAutomaticStrippedStreaming(unsigned int availableRAM, double bias)
 {
   typedef RAMDrivenStrippedStreamingManager<TInputImage> RAMDrivenStrippedStreamingManagerType;
   typename RAMDrivenStrippedStreamingManagerType::Pointer streamingManager = RAMDrivenStrippedStreamingManagerType::New();
   streamingManager->SetAvailableRAMInMB(availableRAM);
-
+  streamingManager->SetBias(bias);
   m_StreamingManager = streamingManager;
 }
 
@@ -119,12 +119,12 @@ StreamingImageVirtualWriter<TInputImage>
 template <class TInputImage>
 void
 StreamingImageVirtualWriter<TInputImage>
-::SetAutomaticTiledStreaming(unsigned int availableRAM)
+::SetAutomaticTiledStreaming(unsigned int availableRAM, double bias)
 {
   typedef RAMDrivenTiledStreamingManager<TInputImage> RAMDrivenTiledStreamingManagerType;
   typename RAMDrivenTiledStreamingManagerType::Pointer streamingManager = RAMDrivenTiledStreamingManagerType::New();
   streamingManager->SetAvailableRAMInMB(availableRAM);
-
+  streamingManager->SetBias(bias);
   m_StreamingManager = streamingManager;
 }