Merge branch 'develop' into data-augmentation

Modules/Core/Streaming/include/otbStreamingManager.h

@@ -88,6 +88,9 @@ public:
    * GetNumberOfSplits() returns. */
   virtual RegionType GetSplit(unsigned int i);
 
+  itkSetMacro(DefaultRAM, MemoryPrintType);
+  itkGetMacro(DefaultRAM, MemoryPrintType);
+
 protected:
   StreamingManager();
   ~StreamingManager() override;
@@ -111,11 +114,13 @@ private:
   StreamingManager(const StreamingManager &); //purposely not implemented
   void operator =(const StreamingManager&);   //purposely not implemented
 
-  /* Compute the available RAM from configuration settings if the input parameter is 0,
-   * otherwise, simply returns the input parameter */
+  /** Compute the available RAM in Bytes from an input value in MByte.
+   *  If the input value is 0, it uses the m_DefaultRAM value.
+   *  If m_DefaultRAM is also 0, it uses the configuration settings */
   MemoryPrintType GetActualAvailableRAMInBytes(MemoryPrintType availableRAMInMB);
 
-
+  /** Default available RAM in MB */
+  MemoryPrintType m_DefaultRAM;
 };
 
 } // End namespace otb

Modules/Core/Streaming/include/otbStreamingManager.txx

@@ -31,6 +31,7 @@ namespace otb
 template <class TImage>
 StreamingManager<TImage>::StreamingManager()
   : m_ComputedNumberOfSplits(0)
+  , m_DefaultRAM(0)
 {
 }
 
@@ -47,8 +48,15 @@ StreamingManager<TImage>::GetActualAvailableRAMInBytes(MemoryPrintType available
 
   if (availableRAMInBytes == 0)
     {
-    // Retrieve it from the configuration
-    availableRAMInBytes = 1024*1024*ConfigurationManager::GetMaxRAMHint();
+    if (m_DefaultRAM != 0)
+      {
+      availableRAMInBytes = 1024*1024*m_DefaultRAM;
+      }
+    else
+      {
+      // Retrieve it from the configuration
+      availableRAMInBytes = 1024*1024*ConfigurationManager::GetMaxRAMHint();
+      }
     }
   return availableRAMInBytes;
 }

Modules/IO/ImageIO/include/otbImageFileWriter.txx

@@ -294,11 +294,17 @@ ImageFileWriter<TInputImage>
       sizemode = m_FilenameHelper->GetStreamingSizeMode();
       }
 
-    double sizevalue = 0.;
+    unsigned int sizevalue = 0;
+    // Save the DefaultRAM value for later
+    unsigned int oldDefaultRAM = m_StreamingManager->GetDefaultRAM();
+    if (sizemode == "auto")
+      {
+      sizevalue = oldDefaultRAM;
+      }
 
     if(m_FilenameHelper->StreamingSizeValueIsSet())
       {
-      sizevalue = m_FilenameHelper->GetStreamingSizeValue();
+      sizevalue = static_cast<unsigned int>(m_FilenameHelper->GetStreamingSizeValue());
       }
 
     if(type == "auto")
@@ -307,7 +313,7 @@ ImageFileWriter<TInputImage>
         {
         otbLogMacro(Warning,<<"In auto streaming type, the sizemode option will be ignored.");
         }
-      if(sizevalue == 0.)
+      if(sizevalue == 0)
         {
         otbLogMacro(Warning,<<"sizemode is auto but sizevalue is 0. Value will be fetched from the OTB_MAX_RAM_HINT environment variable if set, or else use the default value");
         }
@@ -317,7 +323,7 @@ ImageFileWriter<TInputImage>
       {
       if(sizemode == "auto")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"sizemode is auto but sizevalue is 0. Value will be fetched from the OTB_MAX_RAM_HINT environment variable if set, or else use the default value");
           }
@@ -325,27 +331,27 @@ ImageFileWriter<TInputImage>
         }
       else if(sizemode == "nbsplits")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"Streaming sizemode is set to nbsplits but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfDivisionsTiledStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfDivisionsTiledStreaming(sizevalue);
         }
       else if(sizemode == "height")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"Streaming sizemode is set to height but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
 
-        this->SetTileDimensionTiledStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetTileDimensionTiledStreaming(sizevalue);
         }
       }
     else if(type == "stripped")
       {
       if(sizemode == "auto")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"sizemode is auto but sizevalue is 0. Value will be fetched from configuration file if any, or from cmake configuration otherwise.");
           }
@@ -354,30 +360,34 @@ ImageFileWriter<TInputImage>
         }
       else if(sizemode == "nbsplits")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"Streaming sizemode is set to nbsplits but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfDivisionsStrippedStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfDivisionsStrippedStreaming(sizevalue);
         }
       else if(sizemode == "height")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           otbLogMacro(Warning,<<"Streaming sizemode is set to height but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfLinesStrippedStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfLinesStrippedStreaming(sizevalue);
         }
 
       }
     else if (type == "none")
       {
-      if(sizemode!="" || sizevalue!=0.)
+      if(sizemode!="" || sizevalue!=0)
         {
         otbLogMacro(Warning,<<"Streaming is explicitly disabled, sizemode and sizevalue will be ignored.");
         }
       this->SetNumberOfDivisionsTiledStreaming(0);
       }
+
+    // since we change the m_StreamingManager under the hood, we copy the DefaultRAM
+    // value to the new streamingManager.
+    m_StreamingManager->SetDefaultRAM(oldDefaultRAM);
     }
   else
     {
@@ -472,22 +482,19 @@ ImageFileWriter<TInputImage>
   /** Parse region size modes */
   if(m_FilenameHelper->BoxIsSet())
     {
- 	std::vector<int> boxVector;
- 	Utils::ConvertStringToVector( 
- 	m_FilenameHelper->GetBox(), boxVector, "ExtendedFileName:box", ":");
- 	
- 	typename InputImageRegionType::IndexType start;
+    std::vector<int> boxVector;
+    Utils::ConvertStringToVector(
+        m_FilenameHelper->GetBox(), boxVector, "ExtendedFileName:box", ":");
+
+    typename InputImageRegionType::IndexType start;
     typename InputImageRegionType::SizeType  size;
 
     start[0] = boxVector[0];  // first index on X
     start[1] = boxVector[1];  // first index on Y
     size[0]  = boxVector[2];  // size along X
     size[1]  = boxVector[3];  // size along Y
-
     inputRegion.SetSize(size);
-
-    m_ShiftOutputIndex = start;
-    inputRegion.SetIndex(m_ShiftOutputIndex);
+    inputRegion.SetIndex(start);
 
     if (!inputRegion.Crop(inputPtr->GetLargestPossibleRegion()))
       {
@@ -503,6 +510,7 @@ ImageFileWriter<TInputImage>
       }
     otbLogMacro(Info,<<"Writing user defined region ["<<start[0]<<", "<<start[0]+size[0]-1<<"]x["<<start[1]<<", "<<start[1]+size[1]<<"]");
     }
+  m_ShiftOutputIndex = inputRegion.GetIndex();
 
   /**
    * Determine of number of pieces to divide the input.  This will be the
@@ -541,10 +549,11 @@ ImageFileWriter<TInputImage>
   //
   // Setup the ImageIO with information from inputPtr
   //
-  m_ImageIO->SetNumberOfDimensions(TInputImage::ImageDimension);
+  typename TInputImage::PointType origin;
+  inputPtr->TransformIndexToPhysicalPoint(inputRegion.GetIndex(), origin);
   const typename TInputImage::SpacingType&   spacing = inputPtr->GetSpacing();
-  const typename TInputImage::PointType&     origin = inputPtr->GetOrigin();
   const typename TInputImage::DirectionType& direction = inputPtr->GetDirection();
+  m_ImageIO->SetNumberOfDimensions(TInputImage::ImageDimension);
   int direction_sign(0);
   for (unsigned int i = 0; i < TInputImage::ImageDimension; ++i)
     {
@@ -555,7 +564,7 @@ ImageFileWriter<TInputImage>
     // Final image size
     m_ImageIO->SetDimensions(i, inputRegion.GetSize(i));
     m_ImageIO->SetSpacing(i, direction_sign * spacing[i]);
-    m_ImageIO->SetOrigin(i, origin[i] + static_cast<double>(inputRegion.GetIndex()[i]) * spacing[i]);
+    m_ImageIO->SetOrigin(i, origin[i]);
 
     vnl_vector<double> axisDirection(TInputImage::ImageDimension);
     // Please note: direction cosines are stored as columns of the
@@ -618,7 +627,6 @@ ImageFileWriter<TInputImage>
     for (unsigned int i = 0; i < TInputImage::ImageDimension; ++i)
       {
       ioRegion.SetSize(i, streamRegion.GetSize(i));
-      ioRegion.SetIndex(i, streamRegion.GetIndex(i));
       //Set the ioRegion index using the shifted index ( (0,0 without box parameter))
       ioRegion.SetIndex(i, streamRegion.GetIndex(i) - m_ShiftOutputIndex[i]);
       }
@@ -664,7 +672,7 @@ ImageFileWriter<TInputImage>
 
   //Reset global shift on input region (box parameter)
   //It allows calling multiple update over the writer
-  m_ShiftOutputIndex.Fill(0);
+  m_ShiftOutputIndex = inputPtr->GetLargestPossibleRegion().GetIndex();
 }
 
 
@@ -720,10 +728,7 @@ ImageFileWriter<TInputImage>
   InputImageRegionType ioRegion;
 
   // No shift of the ioRegion from the buffered region is expected
-  typename InputImageRegionType::IndexType tmpIndex;
-  tmpIndex.Fill(0);
   itk::ImageIORegionAdaptor<TInputImage::ImageDimension>::
-    //Convert(m_ImageIO->GetIORegion(), ioRegion, tmpIndex);
     Convert(m_ImageIO->GetIORegion(), ioRegion, m_ShiftOutputIndex);
   InputImageRegionType bufferedRegion = input->GetBufferedRegion();
 

Modules/MPI/MPITiffWriter/include/otbSimpleParallelTiffWriter.txx

@@ -315,11 +315,16 @@ SimpleParallelTiffWriter<TInputImage>
       sizemode = m_FilenameHelper->GetStreamingSizeMode();
       }
 
-    double sizevalue = 0.;
-
+    unsigned int sizevalue = 0;
+    // Save the DefaultRAM value for later
+    unsigned int oldDefaultRAM = m_StreamingManager->GetDefaultRAM();
+    if (sizemode == "auto")
+      {
+      sizevalue = oldDefaultRAM;
+      }
     if(m_FilenameHelper->StreamingSizeValueIsSet())
       {
-      sizevalue = m_FilenameHelper->GetStreamingSizeValue();
+      sizevalue = static_cast<unsigned int>(m_FilenameHelper->GetStreamingSizeValue());
       }
 
     if(type == "auto")
@@ -328,7 +333,7 @@ SimpleParallelTiffWriter<TInputImage>
         {
         itkWarningMacro(<<"In auto streaming type, the sizemode option will be ignored.");
         }
-      if(sizevalue == 0.)
+      if(sizevalue == 0)
         {
         itkWarningMacro("sizemode is auto but sizevalue is 0. Value will be fetched from configuration file if any, or from cmake configuration otherwise.");
         }
@@ -338,7 +343,7 @@ SimpleParallelTiffWriter<TInputImage>
       {
       if(sizemode == "auto")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("sizemode is auto but sizevalue is 0. Value will be fetched from configuration file if any, or from cmake configuration otherwise.");
           }
@@ -346,27 +351,27 @@ SimpleParallelTiffWriter<TInputImage>
         }
       else if(sizemode == "nbsplits")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("Streaming sizemode is set to nbsplits but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfDivisionsTiledStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfDivisionsTiledStreaming(sizevalue);
         }
       else if(sizemode == "height")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("Streaming sizemode is set to height but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
 
-        this->SetTileDimensionTiledStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetTileDimensionTiledStreaming(sizevalue);
         }
       }
     else if(type == "stripped")
       {
       if(sizemode == "auto")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("sizemode is auto but sizevalue is 0. Value will be fetched from configuration file if any, or from cmake configuration otherwise.");
           }
@@ -375,30 +380,34 @@ SimpleParallelTiffWriter<TInputImage>
         }
       else if(sizemode == "nbsplits")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("Streaming sizemode is set to nbsplits but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfDivisionsStrippedStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfDivisionsStrippedStreaming(sizevalue);
         }
       else if(sizemode == "height")
         {
-        if(sizevalue == 0.)
+        if(sizevalue == 0)
           {
           itkWarningMacro("Streaming sizemode is set to height but sizevalue is 0. This will result in upredicted behaviour. Please consider setting the sizevalue by using &streaming:sizevalue=x.");
           }
-        this->SetNumberOfLinesStrippedStreaming(static_cast<unsigned int>(sizevalue));
+        this->SetNumberOfLinesStrippedStreaming(sizevalue);
         }
 
       }
     else if (type == "none")
       {
-      if(sizemode!="" || sizevalue!=0.)
+      if(sizemode!="" || sizevalue!=0)
         {
         itkWarningMacro("Streaming is explicitly disabled, sizemode and sizevalue will be ignored.");
         }
       this->SetNumberOfDivisionsTiledStreaming(0);
       }
+
+    // since we change the m_StreamingManager under the hood, we copy the DefaultRAM
+    // value to the new streamingManager.
+    m_StreamingManager->SetDefaultRAM(oldDefaultRAM);
     }
   else
     {
@@ -458,6 +467,7 @@ SimpleParallelTiffWriter<TInputImage>
    */
   inputPtr->UpdateOutputInformation();
   InputImageRegionType inputRegion = inputPtr->GetLargestPossibleRegion();
+  typename InputImageType::PointType origin = inputPtr->GetOrigin();
 
   /** Parse region size modes */
   if(m_FilenameHelper->BoxIsSet())
@@ -497,6 +507,9 @@ SimpleParallelTiffWriter<TInputImage>
       throw e;
       }
     otbMsgDevMacro(<< "inputRegion " << inputRegion);
+
+    // Update the origin
+    inputPtr->TransformIndexToPhysicalPoint(inputRegion.GetIndex(), origin);
     }
 
   // Get number of bands & pixel data type
@@ -538,7 +551,7 @@ SimpleParallelTiffWriter<TInputImage>
   else
     {
     // When mode is not tiled (i.e. striped)
-    block_size_x = inputPtr->GetLargestPossibleRegion().GetSize()[0];
+    block_size_x = inputRegion.GetSize()[0];
     }
 
   // Master process (Rank 0) is responsible for the creation of the output raster.
@@ -546,10 +559,10 @@ SimpleParallelTiffWriter<TInputImage>
     {
     // Set geotransform
     double geotransform[6];
-    geotransform[0] = inputPtr->GetOrigin()[0] - 0.5*inputPtr->GetSignedSpacing()[0];
+    geotransform[0] = origin[0] - 0.5*inputPtr->GetSignedSpacing()[0];
     geotransform[1] = inputPtr->GetSignedSpacing()[0];
     geotransform[2] = 0.0;
-    geotransform[3] = inputPtr->GetOrigin()[1] - 0.5*inputPtr->GetSignedSpacing()[1];
+    geotransform[3] = origin[1] - 0.5*inputPtr->GetSignedSpacing()[1];
     geotransform[4] = 0.0;
     geotransform[5] = inputPtr->GetSignedSpacing()[1];
 
@@ -557,8 +570,8 @@ SimpleParallelTiffWriter<TInputImage>
     if(!m_TiffTiledMode)
       {
       SPTW_ERROR sperr = sptw::create_raster(m_FileName,
-                                             inputPtr->GetLargestPossibleRegion().GetSize()[0],
-                                             inputPtr->GetLargestPossibleRegion().GetSize()[1],
+                                             inputRegion.GetSize()[0],
+                                             inputRegion.GetSize()[1],
                                              nBands,
                                              dataType,
                                              geotransform,
@@ -573,8 +586,8 @@ SimpleParallelTiffWriter<TInputImage>
     else
       {
       SPTW_ERROR sperr = sptw::create_tiled_raster(m_FileName,
-                                                   inputPtr->GetLargestPossibleRegion().GetSize()[0],
-                                                   inputPtr->GetLargestPossibleRegion().GetSize()[1],
+                                                   inputRegion.GetSize()[0],
+                                                   inputRegion.GetSize()[1],
                                                    nBands,
                                                    dataType,
                                                    geotransform,

Modules/Wrappers/ApplicationEngine/src/otbWrapperOutputImageParameter.cxx

@@ -207,6 +207,7 @@ ClampAndWriteVectorImage( itk::ImageBase<2> * in ,
       sptWriter->SetFileName(filename);
       sptWriter->SetInput(clampFilter->GetOutput());
       sptWriter->SetAutomaticAdaptativeStreaming(ramValue);
+      sptWriter->GetStreamingManager()->SetDefaultRAM(ramValue);
       sptWriter->Update();
       }
     
@@ -222,10 +223,9 @@ ClampAndWriteVectorImage( itk::ImageBase<2> * in ,
   
   if(useStandardWriter)
     {
-    
-    writer->SetFileName( filename );                                     
-    writer->SetInput(clampFilter->GetOutput());                                     
-    writer->SetAutomaticAdaptativeStreaming(ramValue);
+    writer->SetFileName( filename );
+    writer->SetInput(clampFilter->GetOutput());
+    writer->GetStreamingManager()->SetDefaultRAM(ramValue);
     writer->Update();
     }
 }
@@ -348,7 +348,7 @@ OutputImageParameter::SwitchRGBAImageWrite()
     {
     m_RGBAUInt8Writer->SetFileName( this->GetFileName() );
     m_RGBAUInt8Writer->SetInput(dynamic_cast<UInt8RGBAImageType*>(m_Image.GetPointer()) );
-    m_RGBAUInt8Writer->SetAutomaticAdaptativeStreaming(m_RAMValue);
+    m_RGBAUInt8Writer->GetStreamingManager()->SetDefaultRAM(m_RAMValue);
     m_RGBAUInt8Writer->Update();
     }
    else
@@ -363,7 +363,7 @@ OutputImageParameter::SwitchRGBImageWrite()
     {
     m_RGBUInt8Writer->SetFileName( this->GetFileName() );
     m_RGBUInt8Writer->SetInput(dynamic_cast<UInt8RGBImageType*>(m_Image.GetPointer()) );
-    m_RGBUInt8Writer->SetAutomaticAdaptativeStreaming(m_RAMValue);
+    m_RGBUInt8Writer->GetStreamingManager()->SetDefaultRAM(m_RAMValue);
     m_RGBUInt8Writer->Update();
     }
    else